draft implement faster auto upload file changed scan

Signed-off-by: Jonas Mayer <jonas.a.mayer@gmx.net>

app/src/androidTest/java/com/owncloud/android/utils/SyncedFolderUtilsTest.kt

@@ -174,7 +174,8 @@ class SyncedFolderUtilsTest : AbstractIT() {
             MediaFolderType.IMAGE,
             false,
             SubFolderRule.YEAR_MONTH,
-            false
+            false,
+            SyncedFolder.NOT_SCANNED_YET
         )
         Assert.assertFalse(SyncedFolderUtils.isQualifyingMediaFolder(folder))
     }
@@ -198,7 +199,8 @@ class SyncedFolderUtilsTest : AbstractIT() {
             MediaFolderType.IMAGE,
             false,
             SubFolderRule.YEAR_MONTH,
-            false
+            false,
+            SyncedFolder.NOT_SCANNED_YET
         )
         Assert.assertFalse(SyncedFolderUtils.isQualifyingMediaFolder(folder))
     }

app/src/main/java/com/nextcloud/client/database/entity/SyncedFolderEntity.kt

@@ -45,6 +45,8 @@ data class SyncedFolderEntity(
     val hidden: Int?,
     @ColumnInfo(name = ProviderTableMeta.SYNCED_FOLDER_SUBFOLDER_RULE)
     val subFolderRule: Int?,
-    @ColumnInfo(name = ProviderTableMeta.SYNCED_EXCLUDE_HIDDEN)
-    val excludeHidden: Int?
+    @ColumnInfo(name = ProviderTableMeta.SYNCED_FOLDER_EXCLUDE_HIDDEN)
+    val excludeHidden: Int?,
+    @ColumnInfo(name = ProviderTableMeta.SYNCED_FOLDER_LAST_SCAN_TIMESTAMP_MS)
+    val lastScanTimestampMs: Long?
 )

app/src/main/java/com/owncloud/android/datamodel/SyncedFolder.java

@@ -21,6 +21,7 @@ import java.io.Serializable;
 public class SyncedFolder implements Serializable, Cloneable {
     public static final long UNPERSISTED_ID = Long.MIN_VALUE;
     public static final long EMPTY_ENABLED_TIMESTAMP_MS = -1;
+    public static final long NOT_SCANNED_YET = -1;
     private static final long serialVersionUID = -793476118299906429L;
 
 
@@ -41,6 +42,7 @@ public class SyncedFolder implements Serializable, Cloneable {
     private boolean hidden;
     private SubFolderRule subfolderRule;
     private boolean excludeHidden;
+    private long lastScanTimestampMs;
 
     /**
      * constructor for new, to be persisted entity.
@@ -75,7 +77,8 @@ public class SyncedFolder implements Serializable, Cloneable {
                         MediaFolderType type,
                         boolean hidden,
                         SubFolderRule subFolderRule,
-                        boolean excludeHidden) {
+                        boolean excludeHidden,
+                        long lastScanTimestampMs) {
         this(UNPERSISTED_ID,
              localPath,
              remotePath,
@@ -91,7 +94,8 @@ public class SyncedFolder implements Serializable, Cloneable {
              type,
              hidden,
              subFolderRule,
-             excludeHidden);
+             excludeHidden,
+             lastScanTimestampMs);
     }
 
     /**
@@ -114,7 +118,8 @@ public class SyncedFolder implements Serializable, Cloneable {
                            MediaFolderType type,
                            boolean hidden,
                            SubFolderRule subFolderRule,
-                           boolean excludeHidden) {
+                           boolean excludeHidden,
+                           long lastScanTimestampMs) {
         this.id = id;
         this.localPath = localPath;
         this.remotePath = remotePath;
@@ -130,6 +135,7 @@ public class SyncedFolder implements Serializable, Cloneable {
         this.hidden = hidden;
         this.subfolderRule = subFolderRule;
         this.excludeHidden = excludeHidden;
+        this.lastScanTimestampMs = lastScanTimestampMs;
     }
 
     /**
@@ -271,4 +277,8 @@ public class SyncedFolder implements Serializable, Cloneable {
     public boolean containsFile(String filePath){
         return filePath.contains(localPath);
     }
+
+    public long getLastScanTimestampMs() { return lastScanTimestampMs; }
+
+    public void setLastScanTimestampMs(long lastScanTimestampMs) { this.lastScanTimestampMs = lastScanTimestampMs; }
 }

app/src/main/java/com/owncloud/android/datamodel/SyncedFolderDisplayItem.java

@@ -60,7 +60,8 @@ public class SyncedFolderDisplayItem extends SyncedFolder {
                                    MediaFolderType type,
                                    boolean hidden,
                                    SubFolderRule subFolderRule,
-                                   boolean excludeHidden) {
+                                   boolean excludeHidden,
+                                   long lastScanTimestampMs) {
         super(id,
               localPath,
               remotePath,
@@ -76,7 +77,8 @@ public class SyncedFolderDisplayItem extends SyncedFolder {
               type,
               hidden,
               subFolderRule,
-              excludeHidden);
+              excludeHidden,
+              lastScanTimestampMs);
         this.filePaths = filePaths;
         this.folderName = folderName;
         this.numberOfFiles = numberOfFiles;
@@ -98,7 +100,8 @@ public class SyncedFolderDisplayItem extends SyncedFolder {
                                    MediaFolderType type,
                                    boolean hidden,
                                    SubFolderRule subFolderRule,
-                                   boolean excludeHidden) {
+                                   boolean excludeHidden,
+                                   long lastScanTimestampMs) {
         super(id,
               localPath,
               remotePath,
@@ -114,7 +117,8 @@ public class SyncedFolderDisplayItem extends SyncedFolder {
               type,
               hidden,
               subFolderRule,
-              excludeHidden);
+              excludeHidden,
+              lastScanTimestampMs);
         this.folderName = folderName;
     }
 

app/src/main/java/com/owncloud/android/datamodel/SyncedFolderProvider.java

@@ -360,7 +360,9 @@ public class SyncedFolderProvider extends Observable {
             SubFolderRule subFolderRule = SubFolderRule.values()[cursor.getInt(
                     cursor.getColumnIndexOrThrow(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_SUBFOLDER_RULE))];
             boolean excludeHidden = cursor.getInt(cursor.getColumnIndexOrThrow(
-                ProviderMeta.ProviderTableMeta.SYNCED_EXCLUDE_HIDDEN)) == 1;
+                ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_EXCLUDE_HIDDEN)) == 1;
+            long lastScanTimestampMs = cursor.getLong(cursor.getColumnIndexOrThrow(
+                ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_LAST_SCAN_TIMESTAMP_MS));
 
 
             syncedFolder = new SyncedFolder(id,
@@ -378,7 +380,8 @@ public class SyncedFolderProvider extends Observable {
                                             type,
                                             hidden,
                                             subFolderRule,
-                                            excludeHidden);
+                                            excludeHidden,
+                                            lastScanTimestampMs);
         }
         return syncedFolder;
     }
@@ -407,8 +410,8 @@ public class SyncedFolderProvider extends Observable {
         cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_TYPE, syncedFolder.getType().id);
         cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_HIDDEN, syncedFolder.isHidden());
         cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_SUBFOLDER_RULE, syncedFolder.getSubfolderRule().ordinal());
-        cv.put(ProviderMeta.ProviderTableMeta.SYNCED_EXCLUDE_HIDDEN, syncedFolder.isExcludeHidden());
-
+        cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_EXCLUDE_HIDDEN, syncedFolder.isExcludeHidden());
+        cv.put(ProviderMeta.ProviderTableMeta.SYNCED_FOLDER_LAST_SCAN_TIMESTAMP_MS, syncedFolder.getLastScanTimestampMs());
         return cv;
     }
 

app/src/main/java/com/owncloud/android/db/ProviderMeta.java

@@ -293,7 +293,8 @@ public class ProviderMeta {
         public static final String SYNCED_FOLDER_NAME_COLLISION_POLICY = "name_collision_policy";
         public static final String SYNCED_FOLDER_HIDDEN = "hidden";
         public static final String SYNCED_FOLDER_SUBFOLDER_RULE = "sub_folder_rule";
-        public static final String SYNCED_EXCLUDE_HIDDEN = "exclude_hidden";
+        public static final String SYNCED_FOLDER_EXCLUDE_HIDDEN = "exclude_hidden";
+        public static final String SYNCED_FOLDER_LAST_SCAN_TIMESTAMP_MS = "last_scan_timestamp_ms";
 
         // Columns of external links table
         public static final String EXTERNAL_LINKS_ICON_URL = "icon_url";

app/src/main/java/com/owncloud/android/ui/activity/SyncedFoldersActivity.kt

@@ -387,7 +387,8 @@ class SyncedFoldersActivity :
             syncedFolder.type,
             syncedFolder.isHidden,
             syncedFolder.subfolderRule,
-            syncedFolder.isExcludeHidden
+            syncedFolder.isExcludeHidden,
+            syncedFolder.lastScanTimestampMs
         )
     }
 
@@ -418,7 +419,8 @@ class SyncedFoldersActivity :
             mediaFolder.type,
             syncedFolder.isHidden,
             syncedFolder.subfolderRule,
-            syncedFolder.isExcludeHidden
+            syncedFolder.isExcludeHidden,
+            syncedFolder.lastScanTimestampMs
         )
     }
 
@@ -448,7 +450,8 @@ class SyncedFoldersActivity :
             mediaFolder.type,
             false,
             SubFolderRule.YEAR_MONTH,
-            false
+            false,
+            SyncedFolder.NOT_SCANNED_YET
         )
     }
 
@@ -541,7 +544,8 @@ class SyncedFoldersActivity :
                         MediaFolderType.CUSTOM,
                         false,
                         SubFolderRule.YEAR_MONTH,
-                        false
+                        false,
+                        SyncedFolder.NOT_SCANNED_YET
                     )
                     onSyncFolderSettingsClick(0, emptyCustomFolder)
                 } else {
@@ -658,7 +662,8 @@ class SyncedFoldersActivity :
                 syncedFolder.type,
                 syncedFolder.isHidden,
                 syncedFolder.subFolderRule,
-                syncedFolder.isExcludeHidden
+                syncedFolder.isExcludeHidden,
+                SyncedFolder.NOT_SCANNED_YET
             )
             saveOrUpdateSyncedFolder(newCustomFolder)
             adapter.addSyncFolderItem(newCustomFolder)

app/src/main/java/com/owncloud/android/utils/FilesSyncHelper.java

@@ -55,6 +55,59 @@ public final class FilesSyncHelper {
         // utility class -> private constructor
     }
 
+    private static void insertCustomFolderIntoDB(Path path,
+                                                 SyncedFolder syncedFolder,
+                                                 FilesystemDataProvider filesystemDataProvider,
+                                                 long lastCheck,
+                                                 long thisCheck) {
+
+        final long enabledTimestampMs = syncedFolder.getEnabledTimestampMs();
+
+        try {
+            FileUtil.walkFileTree(path, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
+                    File file = path.toFile();
+                    if (syncedFolder.isExcludeHidden() && file.isHidden()) {
+                        // exclude hidden file or folder
+                        return FileVisitResult.CONTINUE;
+                    }
+
+                    if (lastCheck != SyncedFolder.NOT_SCANNED_YET && attrs.lastModifiedTime().toMillis() < lastCheck) {
+                        // skip files that were already checked
+                        return FileVisitResult.CONTINUE;
+                    }
+
+                    if (syncedFolder.isExisting() || attrs.lastModifiedTime().toMillis() >= enabledTimestampMs) {
+                        // storeOrUpdateFileValue takes a few ms
+                        // -> Rest of this file check takes not even 1 ms.
+                        filesystemDataProvider.storeOrUpdateFileValue(path.toAbsolutePath().toString(),
+                                                                      attrs.lastModifiedTime().toMillis(),
+                                                                      file.isDirectory(), syncedFolder);
+                    }
+
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                    if (syncedFolder.isExcludeHidden() && dir.compareTo(Paths.get(syncedFolder.getLocalPath())) != 0 && dir.toFile().isHidden()) {
+                        return null;
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException exc) {
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+            syncedFolder.setLastScanTimestampMs(thisCheck);
+        } catch (IOException e) {
+            Log_OC.e(TAG, "Something went wrong while indexing files for auto upload", e);
+        }
+    }
+
     private static void insertAllDBEntriesForSyncedFolder(SyncedFolder syncedFolder) {
         final Context context = MainApp.getAppContext();
         final ContentResolver contentResolver = context.getContentResolver();
@@ -63,92 +116,31 @@ public final class FilesSyncHelper {
 
         if (syncedFolder.isEnabled() && (syncedFolder.isExisting() || enabledTimestampMs >= 0)) {
             MediaFolderType mediaType = syncedFolder.getType();
+            final long lastCheck = syncedFolder.getLastScanTimestampMs();
+            final long thisCheck = System.currentTimeMillis();
+
             if (mediaType == MediaFolderType.IMAGE) {
-                FilesSyncHelper.insertContentIntoDB(MediaStore.Images.Media.INTERNAL_CONTENT_URI
-                    , syncedFolder);
+                FilesSyncHelper.insertContentIntoDB(MediaStore.Images.Media.INTERNAL_CONTENT_URI,
+                                                    syncedFolder,
+                                                    lastCheck, thisCheck);
                 FilesSyncHelper.insertContentIntoDB(MediaStore.Images.Media.EXTERNAL_CONTENT_URI,
-                                                    syncedFolder);
+                                                    syncedFolder,
+                                                    lastCheck, thisCheck);
             } else if (mediaType == MediaFolderType.VIDEO) {
                 FilesSyncHelper.insertContentIntoDB(MediaStore.Video.Media.INTERNAL_CONTENT_URI,
-                                                    syncedFolder);
+                                                    syncedFolder,
+                                                    lastCheck, thisCheck);
                 FilesSyncHelper.insertContentIntoDB(MediaStore.Video.Media.EXTERNAL_CONTENT_URI,
-                                                    syncedFolder);
+                                                    syncedFolder,
+                                                    lastCheck, thisCheck);
             } else {
-                try {
                     FilesystemDataProvider filesystemDataProvider = new FilesystemDataProvider(contentResolver);
                     Path path = Paths.get(syncedFolder.getLocalPath());
 
                     long startTime = System.nanoTime();
-                    // chick check for changes
-                    long lastCheck = System.currentTimeMillis();
-                    ArrayList<File> changedFiles = new ArrayList<>();
-                    FileUtil.walkFileTree(path, new SimpleFileVisitor<Path>() {
-                        @Override
-                        public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
-                            File file = path.toFile();
-                            if (syncedFolder.isExcludeHidden() && file.isHidden()) {
-                                // exclude hidden file or folder
-                                return FileVisitResult.CONTINUE;
-                            }
-                            if (file.lastModified() >= lastCheck){
-                                changedFiles.add(file);
-                            }
-
-                            return FileVisitResult.CONTINUE;
-                        }
-
-                        @Override
-                        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
-                            if (syncedFolder.isExcludeHidden() && dir.compareTo(Paths.get(syncedFolder.getLocalPath())) != 0 && dir.toFile().isHidden()) {
-                                return null;
-                            }
-                            return FileVisitResult.CONTINUE;
-                        }
-
-                        @Override
-                        public FileVisitResult visitFileFailed(Path file, IOException exc) {
-                            return FileVisitResult.CONTINUE;
-                        }
-                    });
-                    Log_OC.d(TAG,"FILESYNC FINISHED QUICK CHECK FILE "+path+" "+(System.nanoTime() - startTime));
-                    startTime = System.nanoTime();
-                    FileUtil.walkFileTree(path, new SimpleFileVisitor<Path>() {
-                        @Override
-                        public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
-                            File file = path.toFile();
-                            if (syncedFolder.isExcludeHidden() && file.isHidden()) {
-                                // exclude hidden file or folder
-                                return FileVisitResult.CONTINUE;
-                            }
-                            if (syncedFolder.isExisting() || attrs.lastModifiedTime().toMillis() >= enabledTimestampMs) {
-                                // storeOrUpdateFileValue takes a few millisec
-                                // -> Rest of this file check takes not even 1 millisec.
-                                filesystemDataProvider.storeOrUpdateFileValue(path.toAbsolutePath().toString(),
-                                                                              attrs.lastModifiedTime().toMillis(),
-                                                                              file.isDirectory(), syncedFolder);
-                            }
-
-                            return FileVisitResult.CONTINUE;
-                        }
-
-                        @Override
-                        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
-                            if (syncedFolder.isExcludeHidden() && dir.compareTo(Paths.get(syncedFolder.getLocalPath())) != 0 && dir.toFile().isHidden()) {
-                                return null;
-                            }
-                            return FileVisitResult.CONTINUE;
-                        }
-
-                        @Override
-                        public FileVisitResult visitFileFailed(Path file, IOException exc) {
-                            return FileVisitResult.CONTINUE;
-                        }
-                    });
-                    Log_OC.d(TAG,"FILESYNC FINISHED LONG CHECK FILE "+path+" "+(System.nanoTime() - startTime));
-
-                } catch (IOException e) {
-                    Log_OC.e(TAG, "Something went wrong while indexing files for auto upload", e);
-                }
+                    FilesSyncHelper.insertCustomFolderIntoDB(path, syncedFolder, filesystemDataProvider, lastCheck, thisCheck);
+                    Log_OC.d(TAG,"FILESYNC FINISHED LONG CHECK FOLDER "+path+" "+(System.nanoTime() - startTime));
+
             }
         }
     }
@@ -200,7 +192,7 @@ public final class FilesSyncHelper {
         return filePath;
     }
 
-    private static void insertContentIntoDB(Uri uri, SyncedFolder syncedFolder) {
+    private static void insertContentIntoDB(Uri uri, SyncedFolder syncedFolder, long lastCheckMs, long thisCheckMs) {
         final Context context = MainApp.getAppContext();
         final ContentResolver contentResolver = context.getContentResolver();
 
@@ -231,15 +223,23 @@ public final class FilesSyncHelper {
             column_index_date_modified = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns.DATE_MODIFIED);
             while (cursor.moveToNext()) {
                 contentPath = cursor.getString(column_index_data);
-                Log_OC.d(TAG,"FILESYNC CHECK File "+contentPath);
                 isFolder = new File(contentPath).isDirectory();
+
+                if (syncedFolder.getLastScanTimestampMs() != SyncedFolder.NOT_SCANNED_YET &&
+                    cursor.getLong(column_index_date_modified) < (lastCheckMs / 1000.0)) {
+                    continue;
+                }
+
                 if (syncedFolder.isExisting() || cursor.getLong(column_index_date_modified) >= enabledTimestampMs / 1000.0) {
+                    // storeOrUpdateFileValue takes a few ms
+                    // -> Rest of this file check takes not even 1 ms.
                     filesystemDataProvider.storeOrUpdateFileValue(contentPath,
                                                                   cursor.getLong(column_index_date_modified), isFolder,
                                                                   syncedFolder);
                 }
             }
             cursor.close();
+            syncedFolder.setLastScanTimestampMs(thisCheckMs);
         }
     }