Proof of concept for 10x faster check for changed files

Signed-off-by: Jonas Mayer <jonas.a.mayer@gmx.net>

app/src/main/java/com/nextcloud/client/jobs/FilesSyncWork.kt

@@ -70,7 +70,7 @@ class FilesSyncWork(
 
     @Suppress("MagicNumber")
     private fun updateForegroundWorker(progressPercent: Int, useForegroundWorker: Boolean) {
-        if (useForegroundWorker) {
+        if (!useForegroundWorker) {
             return
         }
 
@@ -95,6 +95,7 @@ class FilesSyncWork(
     @Suppress("MagicNumber")
     override fun doWork(): Result {
         backgroundJobManager.logStartOfWorker(BackgroundJobManagerImpl.formatClassTag(this::class))
+        Log_OC.d(TAG,"FILESYNC WORKER STARTED")
 
         val overridePowerSaving = inputData.getBoolean(OVERRIDE_POWER_SAVING, false)
         // If we are in power save mode, better to postpone upload
@@ -114,7 +115,9 @@ class FilesSyncWork(
 
         // Get changed files from ContentObserverWork (only images and videos) or by scanning filesystem
         val changedFiles = inputData.getStringArray(CHANGED_FILES)
+        Log_OC.d(TAG,"FILESYNC WORKER CHANGED FILES: "+changedFiles.contentToString())
         collectChangedFiles(changedFiles)
+        Log_OC.d(TAG,"FILESYNC WORKER CHECKED CHANGED FILES")
 
         // Create all the providers we'll need
         val filesystemDataProvider = FilesystemDataProvider(contentResolver)
@@ -129,11 +132,7 @@ class FilesSyncWork(
                 (50 + (index.toDouble() / syncedFolders.size.toDouble()) * 50).toInt(),
                 changedFiles.isNullOrEmpty()
             )
-            if (syncedFolder.isEnabled && (
-                    changedFiles.isNullOrEmpty() ||
-                        MediaFolderType.CUSTOM != syncedFolder.type
-                    )
-            ) {
+            if (syncedFolder.isEnabled) {
                 syncFolder(
                     context,
                     resources,
@@ -145,6 +144,7 @@ class FilesSyncWork(
                 )
             }
         }
+        Log_OC.d(TAG,"FILESYNC WORKER ENDED")
         val result = Result.success()
         backgroundJobManager.logEndOfWorker(BackgroundJobManagerImpl.formatClassTag(this::class), result)
         return result
@@ -222,7 +222,7 @@ class FilesSyncWork(
             needsWifi = syncedFolder.isWifiOnly
             uploadAction = syncedFolder.uploadAction
         }
-
+        Log_OC.d(TAG,"FILESYNC SCHEDULE UPLOAD OF FILE")
         FileUploadHelper.instance().uploadNewFiles(
             user,
             localPaths,

app/src/main/java/com/owncloud/android/datamodel/FilesystemDataProvider.java

@@ -107,6 +107,7 @@ public class FilesystemDataProvider {
 
     public void storeOrUpdateFileValue(String localPath, long modifiedAt, boolean isFolder, SyncedFolder syncedFolder) {
 
+        // takes multiple milliseconds to query data from database (around 75% of execution time) (6ms)
         FileSystemDataSet data = getFilesystemDataSet(localPath, syncedFolder);
 
         int isFolderValue = 0;
@@ -145,7 +146,7 @@ public class FilesystemDataProvider {
                 }
             }
 
-
+            // updating data takes multiple milliseconds (around 25% of exec time) (2 ms)
             int result = contentResolver.update(
                 ProviderMeta.ProviderTableMeta.CONTENT_URI_FILESYSTEM,
                 cv,

app/src/main/java/com/owncloud/android/utils/FilesSyncHelper.java

@@ -20,7 +20,6 @@ import com.nextcloud.client.device.BatteryStatus;
 import com.nextcloud.client.device.PowerManagementService;
 import com.nextcloud.client.jobs.BackgroundJobManager;
 import com.nextcloud.client.jobs.upload.FileUploadHelper;
-import com.nextcloud.client.jobs.upload.FileUploadWorker;
 import com.nextcloud.client.network.ConnectivityService;
 import com.owncloud.android.MainApp;
 import com.owncloud.android.datamodel.FilesystemDataProvider;
@@ -40,6 +39,7 @@ import org.lukhnos.nnio.file.attribute.BasicFileAttributes;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 
 import static com.owncloud.android.datamodel.OCFile.PATH_SEPARATOR;
 
@@ -78,6 +78,40 @@ public final class FilesSyncHelper {
                     FilesystemDataProvider filesystemDataProvider = new FilesystemDataProvider(contentResolver);
                     Path path = Paths.get(syncedFolder.getLocalPath());
 
+                    long startTime = System.nanoTime();
+                    // chick check for changes
+                    long lastCheck = System.currentTimeMillis();
+                    ArrayList<File> changedFiles = new ArrayList<>();
+                    FileUtil.walkFileTree(path, new SimpleFileVisitor<Path>() {
+                        @Override
+                        public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
+                            File file = path.toFile();
+                            if (syncedFolder.isExcludeHidden() && file.isHidden()) {
+                                // exclude hidden file or folder
+                                return FileVisitResult.CONTINUE;
+                            }
+                            if (file.lastModified() >= lastCheck){
+                                changedFiles.add(file);
+                            }
+
+                            return FileVisitResult.CONTINUE;
+                        }
+
+                        @Override
+                        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+                            if (syncedFolder.isExcludeHidden() && dir.compareTo(Paths.get(syncedFolder.getLocalPath())) != 0 && dir.toFile().isHidden()) {
+                                return null;
+                            }
+                            return FileVisitResult.CONTINUE;
+                        }
+
+                        @Override
+                        public FileVisitResult visitFileFailed(Path file, IOException exc) {
+                            return FileVisitResult.CONTINUE;
+                        }
+                    });
+                    Log_OC.d(TAG,"FILESYNC FINISHED QUICK CHECK FILE "+path+" "+(System.nanoTime() - startTime));
+                    startTime = System.nanoTime();
                     FileUtil.walkFileTree(path, new SimpleFileVisitor<Path>() {
                         @Override
                         public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
@@ -87,6 +121,8 @@ public final class FilesSyncHelper {
                                 return FileVisitResult.CONTINUE;
                             }
                             if (syncedFolder.isExisting() || attrs.lastModifiedTime().toMillis() >= enabledTimestampMs) {
+                                // storeOrUpdateFileValue takes a few millisec
+                                // -> Rest of this file check takes not even 1 millisec.
                                 filesystemDataProvider.storeOrUpdateFileValue(path.toAbsolutePath().toString(),
                                                                               attrs.lastModifiedTime().toMillis(),
                                                                               file.isDirectory(), syncedFolder);
@@ -108,6 +144,8 @@ public final class FilesSyncHelper {
                             return FileVisitResult.CONTINUE;
                         }
                     });
+                    Log_OC.d(TAG,"FILESYNC FINISHED LONG CHECK FILE "+path+" "+(System.nanoTime() - startTime));
+
                 } catch (IOException e) {
                     Log_OC.e(TAG, "Something went wrong while indexing files for auto upload", e);
                 }
@@ -117,6 +155,7 @@ public final class FilesSyncHelper {
 
     public static void insertAllDBEntries(SyncedFolderProvider syncedFolderProvider) {
         for (SyncedFolder syncedFolder : syncedFolderProvider.getSyncedFolders()) {
+            Log_OC.d(TAG,"FILESYNC CHECK FOLDER "+syncedFolder.getLocalPath());
             if (syncedFolder.isEnabled()) {
                 insertAllDBEntriesForSyncedFolder(syncedFolder);
             }
@@ -135,6 +174,7 @@ public final class FilesSyncHelper {
                     filesystemDataProvider.storeOrUpdateFileValue(changedFile,
                                                                   file.lastModified(),file.isDirectory(),
                                                                   syncedFolder);
+                    Log_OC.d(TAG,"FILESYNC ADDED UPLOAD TO DB");
                     break;
                 }
             }
@@ -191,6 +231,7 @@ public final class FilesSyncHelper {
             column_index_date_modified = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns.DATE_MODIFIED);
             while (cursor.moveToNext()) {
                 contentPath = cursor.getString(column_index_data);
+                Log_OC.d(TAG,"FILESYNC CHECK File "+contentPath);
                 isFolder = new File(contentPath).isDirectory();
                 if (syncedFolder.isExisting() || cursor.getLong(column_index_date_modified) >= enabledTimestampMs / 1000.0) {
                     filesystemDataProvider.storeOrUpdateFileValue(contentPath,