api controller refactor

app/assets/javascripts/app/frontend/controllers/_base.js

@@ -1,9 +1,9 @@
 class BaseCtrl {
-  constructor($rootScope, modelManager, apiController, dbManager) {
+  constructor(syncManager, dbManager) {
     dbManager.openDatabase(null, function(){
       // new database, delete syncToken so that items can be refetched entirely from server
-      apiController.clearSyncToken();
+      syncManager.clearSyncToken();
-      apiController.sync();
+      syncManager.sync();
     })
   }
 }

app/assets/javascripts/app/frontend/controllers/header.js

@@ -16,7 +16,7 @@ angular.module('app.frontend')
       }
     }
   })
-  .controller('HeaderCtrl', function (apiController, modelManager, $timeout, dbManager) {
+  .controller('HeaderCtrl', function (apiController, modelManager, $timeout, dbManager, syncManager) {
 
     this.user = apiController.user;
 
@@ -43,7 +43,7 @@ angular.module('app.frontend')
 
     this.refreshData = function() {
       this.isRefreshing = true;
-      apiController.sync(function(response){
+      syncManager.sync(function(response){
         $timeout(function(){
           this.isRefreshing = false;
         }.bind(this), 200)

app/assets/javascripts/app/frontend/controllers/home.js

@@ -1,14 +1,14 @@
 angular.module('app.frontend')
-.controller('HomeCtrl', function ($scope, $rootScope, $timeout, apiController, modelManager) {
+.controller('HomeCtrl', function ($scope, $rootScope, $timeout, modelManager, syncManager) {
     $rootScope.bodyClass = "app-body-class";
 
-    apiController.loadLocalItems(function(items){
+    syncManager.loadLocalItems(function(items){
       $scope.$apply();
 
-      apiController.sync(null);
+      syncManager.sync(null);
       // refresh every 30s
       setInterval(function () {
-        apiController.sync(null);
+        syncManager.sync(null);
       }, 30000);
     });
 
@@ -31,7 +31,7 @@ angular.module('app.frontend')
         modelManager.createRelationshipBetweenItems(note, tag);
       }
 
-      apiController.sync();
+      syncManager.sync();
     }
 
     /*
@@ -57,7 +57,7 @@ angular.module('app.frontend')
 
     $scope.tagsSave = function(tag, callback) {
       tag.setDirty(true);
-      apiController.sync(callback);
+      syncManager.sync(callback);
     }
 
     /*
@@ -69,7 +69,7 @@ angular.module('app.frontend')
       if(validNotes == 0) {
         modelManager.setItemToBeDeleted(tag);
         // if no more notes, delete tag
-        apiController.sync(function(){
+        syncManager.sync(function(){
           // force scope tags to update on sub directives
           $scope.tags = [];
           $timeout(function(){
@@ -100,7 +100,7 @@ angular.module('app.frontend')
     $scope.saveNote = function(note, callback) {
       note.setDirty(true);
 
-      apiController.sync(function(response){
+      syncManager.sync(function(response){
         if(response && response.error) {
           if(!$scope.didShowErrorAlert) {
             $scope.didShowErrorAlert = true;
@@ -137,8 +137,8 @@ angular.module('app.frontend')
         return;
       }
 
-      apiController.sync(function(){
+      syncManager.sync(function(){
-        if(!apiController.user) {
+        if(syncManager.offline) {
           // when deleting items while ofline, we need to explictly tell angular to refresh UI
           setTimeout(function () {
             $scope.safeApply();

app/assets/javascripts/app/frontend/models/local/itemParams.js

@@ -0,0 +1,52 @@
+class ItemParams {
+
+  constructor(item, ek, encryptionHelper) {
+    this.item = item;
+    this.ek = ek;
+    this.encryptionHelper = encryptionHelper;
+  }
+
+  paramsForExportFile() {
+    this.additionalFields = ["created_at", "updated_at"];
+    this.forExportFile = true;
+    return _.omit(this.__params(), ["deleted"]);
+  }
+
+  paramsForExtension() {
+    return this.paramsForExportFile();
+  }
+
+  paramsForSync() {
+    return __params(null, false);
+  }
+
+  __params() {
+    var itemCopy = _.cloneDeep(this.item);
+
+    console.assert(!item.dummy, "Item is dummy, should not have gotten here.", item.dummy)
+
+    var params = {uuid: item.uuid, content_type: item.content_type, deleted: item.deleted};
+
+    if(this.ek) {
+      this.encryptionHelper.encryptItem(itemCopy, this.ek);
+      params.content = itemCopy.content;
+      params.enc_item_key = itemCopy.enc_item_key;
+      params.auth_hash = itemCopy.auth_hash;
+    }
+    else {
+      params.content = this.forExportFile ? itemCopy.createContentJSONFromProperties() : "000" + Neeto.crypto.base64(JSON.stringify(itemCopy.createContentJSONFromProperties()));
+      if(!this.forExportFile) {
+        params.enc_item_key = null;
+        params.auth_hash = null;
+      }
+    }
+
+    if(this.additionalFields) {
+      _.merge(params, _.pick(item, this.additionalFields));
+    }
+
+    return params;
+  }
+
+
+}

app/assets/javascripts/app/frontend/models/sync/syncProvider.js

@@ -1,38 +0,0 @@
-class SyncProvider {
-  constructor(obj) {
-    this.encrypted = true;
-    _.merge(this, obj);
-  }
-
-  addPendingItems(items) {
-    if(!this.pendingItems) {
-      this.pendingItems = [];
-    }
-
-    this.pendingItems = this.pendingItems.concat(items);
-  }
-
-  removePendingItems(items) {
-    this.pendingItems = _.difference(this.pendingItems, items);
-  }
-
-  get status() {
-    if(!this.enabled) {
-      return null;
-    }
-
-    if(this.primary) return "primary";
-    else return "secondary";
-  }
-
-  asJSON() {
-    return {
-      enabled: this.enabled,
-      url: this.url,
-      primary: this.primary,
-      keyName: this.keyName,
-      syncToken: this.syncToken
-    }
-  }
-
-}

app/assets/javascripts/app/services/apiController.js

@@ -87,12 +87,7 @@ angular.module('app.frontend')
             var params = {password: keys.pw, email: email};
             _.merge(request, params);
             request.post().then(function(response){
-              localStorage.setItem("server", url);
+              this.handleAuthResponse(response, url, authParams, mk);
-              localStorage.setItem("jwt", response.token);
-              localStorage.setItem("user", JSON.stringify(response.user));
-              localStorage.setItem("auth_params", JSON.stringify(authParams));
-              keyManager.addKey(SNKeyName, mk);
-              this.addStandardFileSyncProvider(url);
               callback(response);
             }.bind(this))
             .catch(function(response){
@@ -103,6 +98,15 @@ angular.module('app.frontend')
         }.bind(this))
       }
 
+      this.handleAuthResponse = function(response, url, authParams, mk) {
+        localStorage.setItem("server", url);
+        localStorage.setItem("jwt", response.token);
+        localStorage.setItem("user", JSON.stringify(response.user));
+        localStorage.setItem("auth_params", JSON.stringify(_.omit(authParams, ["pw_nonce"])));
+        syncManager.addKey(syncManager.SNKeyName, mk);
+        syncManager.addStandardFileSyncProvider(url);
+      }
+
       this.register = function(url, email, password, callback) {
         Neeto.crypto.generateInitialEncryptionKeysForUser({password: password, email: email}, function(keys, authParams){
           var mk = keys.mk;
@@ -111,12 +115,7 @@ angular.module('app.frontend')
           var params = _.merge({password: keys.pw, email: email}, authParams);
           _.merge(request, params);
           request.post().then(function(response){
-            localStorage.setItem("server", url);
+            this.handleAuthResponse(response, url, authParams, mk);
-            localStorage.setItem("jwt", response.token);
-            localStorage.setItem("user", JSON.stringify(response.user));
-            localStorage.setItem("auth_params", JSON.stringify(_.omit(authParams, ["pw_nonce"])));
-            keyManager.addKey(SNKeyName, mk);
-            this.addStandardFileSyncProvider(url);
             callback(response);
           }.bind(this))
           .catch(function(response){
@@ -181,224 +180,11 @@ angular.module('app.frontend')
       Sync
       */
 
-      this.syncWithOptions = function(callback, options = {}) {
-
-        var allDirtyItems = modelManager.getDirtyItems();
-
-        // we want to write all dirty items to disk only if the user is not signed in, or if the sync op fails
-        // if the sync op succeeds, these items will be written to disk by handling the "saved_items" response from the server
-        var writeAllDirtyItemsToDisk = function(completion) {
-          this.writeItemsToLocalStorage(allDirtyItems, function(responseItems){
-            if(completion) {
-              completion();
-            }
-          })
-        }.bind(this);
-
-        if(!this.isUserSignedIn()) {
-          writeAllDirtyItemsToDisk(function(){
-            // delete anything needing to be deleted
-            allDirtyItems.forEach(function(item){
-              if(item.deleted) {
-                modelManager.removeItemLocally(item);
-              }
-            }.bind(this))
-
-            modelManager.clearDirtyItems(allDirtyItems);
-
-          }.bind(this))
-
-          this.syncOpInProgress = false;
-
-          if(this.syncProviders.length == 0 && callback) {
-            callback();
-          }
-        }
-
-        for(let provider of this.syncProviders) {
-          if(!provider.enabled) {
-            continue;
-          }
-          provider.addPendingItems(allDirtyItems);
-          this.__performSyncWithProvider(provider, options, function(response){
-            if(provider.primary) {
-              if(callback) {
-                callback(response)
-              }
-            }
-          })
-        }
-
-        modelManager.clearDirtyItems(allDirtyItems);
-      }
-
-      this.__performSyncWithProvider = function(provider, options, callback) {
-        if(provider.syncOpInProgress) {
-          provider.repeatOnCompletion = true;
-          console.log("Sync op in progress for provider; returning.", provider);
-          return;
-        }
-
-
-        provider.syncOpInProgress = true;
-
-        let submitLimit = 100;
-        var allItems = provider.pendingItems;
-        var subItems = allItems.slice(0, submitLimit);
-        if(subItems.length < allItems.length) {
-          // more items left to be synced, repeat
-          provider.repeatOnCompletion = true;
-        } else {
-          provider.repeatOnCompletion = false;
-        }
-
-        console.log("Syncing with provider", provider, subItems);
-
-        // Remove dirty items now. If this operation fails, we'll re-add them.
-        // This allows us to queue changes on the same item
-        provider.removePendingItems(subItems);
-
-        var request = Restangular.oneUrl(provider.url, provider.url);
-        request.limit = 150;
-        request.items = _.map(subItems, function(item){
-          return this.paramsForItem(item, provider.encrypted, provider.ek, options.additionalFields, false);
-        }.bind(this));
-
-        request.sync_token = provider.syncToken;
-        request.cursor_token = provider.cursorToken;
-
-        request.post().then(function(response) {
-
-          console.log("Sync completion", response);
-
-          provider.syncToken = response.sync_token;
-
-          if(provider.primary) {
-            $rootScope.$broadcast("sync:updated_token", provider.syncToken);
-
-            // handle cursor token (more results waiting, perform another sync)
-            provider.cursorToken = response.cursor_token;
-
-            var retrieved = this.handleItemsResponse(response.retrieved_items, null, provider);
-            // merge only metadata for saved items
-            var omitFields = ["content", "auth_hash"];
-            var saved = this.handleItemsResponse(response.saved_items, omitFields, provider);
-
-            this.handleUnsavedItemsResponse(response.unsaved, provider)
-
-            this.writeItemsToLocalStorage(saved, null);
-            this.writeItemsToLocalStorage(retrieved, null);
-          }
-
-          provider.syncOpInProgress = false;
-          this.didMakeChangesToSyncProviders();
-
-          if(provider.cursorToken || provider.repeatOnCompletion == true) {
-            this.__performSyncWithProvider(provider, options, callback);
-          } else {
-            if(callback) {
-              callback(response);
-            }
-          }
-
-        }.bind(this))
-        .catch(function(response){
-          console.log("Sync error: ", response);
-
-          // Re-add subItems since this operation failed. We'll have to try again.
-          provider.addPendingItems(subItems);
-          provider.syncOpInProgress = false;
-
-          if(provider.primary) {
-            this.writeItemsToLocalStorage(allItems, null);
-          }
-
-          if(callback) {
-            callback({error: "Sync error"});
-          }
-        }.bind(this))
-      }
-
-      this.sync = function(callback) {
-        this.syncWithOptions(callback, undefined);
-      }
-
-      this.handleUnsavedItemsResponse = function(unsaved, provider) {
-        if(unsaved.length == 0) {
-          return;
-        }
-
-        console.log("Handle unsaved", unsaved);
-        for(var mapping of unsaved) {
-          var itemResponse = mapping.item;
-          var item = modelManager.findItem(itemResponse.uuid);
-          var error = mapping.error;
-          if(error.tag == "uuid_conflict") {
-              item.alternateUUID();
-              item.setDirty(true);
-              item.markAllReferencesDirty();
-          }
-        }
-
-        this.__performSyncWithProvider(provider, {additionalFields: ["created_at", "updated_at"]}, null);
-      }
-
-      this.handleItemsResponse = function(responseItems, omitFields, syncProvider) {
-        var ek = syncProvider ? keyManager.keyForName(syncProvider.keyName).key : null;
-        this.decryptItemsWithKey(responseItems, ek);
-        return modelManager.mapResponseItemsToLocalModelsOmittingFields(responseItems, omitFields);
-      }
-
-      this.paramsForExportFile = function(item, ek, encrypted) {
-        return _.omit(this.paramsForItem(item, encrypted, ek, ["created_at", "updated_at"], true), ["deleted"]);
-      }
-
-      this.paramsForExtension = function(item, encrypted) {
-        return _.omit(this.paramsForItem(item, encrypted, ["created_at", "updated_at"], true), ["deleted"]);
-      }
-
-      this.paramsForItem = function(item, encrypted, ek, additionalFields, forExportFile) {
-        var itemCopy = _.cloneDeep(item);
-
-        if(encrypted) {
-          console.assert(ek.length, "Attempting to encrypt without encryption key.");
-        }
-        console.assert(!item.dummy, "Item is dummy, should not have gotten here.", item.dummy)
-
-        var params = {uuid: item.uuid, content_type: item.content_type, deleted: item.deleted};
-
-        if(encrypted) {
-          this.encryptSingleItem(itemCopy, ek);
-          params.content = itemCopy.content;
-          params.enc_item_key = itemCopy.enc_item_key;
-          params.auth_hash = itemCopy.auth_hash;
-        }
-        else {
-          params.content = forExportFile ? itemCopy.createContentJSONFromProperties() : "000" + Neeto.crypto.base64(JSON.stringify(itemCopy.createContentJSONFromProperties()));
-          if(!forExportFile) {
-            params.enc_item_key = null;
-            params.auth_hash = null;
-          }
-        }
-
-        if(additionalFields) {
-          _.merge(params, _.pick(item, additionalFields));
-        }
-
-        return params;
-      }
 
       /*
       Import
       */
 
-      this.clearSyncToken = function() {
-        var primary = this.primarySyncProvider();
-        if(primary) {
-          primary.syncToken = null;
-        }
-      }
-
       this.importJSONData = function(data, password, callback) {
         console.log("Importing data", data);
 
@@ -439,7 +225,7 @@ angular.module('app.frontend')
       Export
       */
 
-      this.itemsDataFile = function(encrypted, custom_ek) {
+      this.itemsDataFile = function(ek) {
         var textFile = null;
         var makeTextFile = function (text) {
           var data = new Blob([text], {type: 'text/json'});
@@ -456,20 +242,16 @@ angular.module('app.frontend')
           return textFile;
         }.bind(this);
 
-        var ek = custom_ek;
-        if(encrypted && !custom_ek) {
-          ek = this.retrieveMk();
-        }
-
         var items = _.map(modelManager.allItemsMatchingTypes(["Tag", "Note"]), function(item){
-          return this.paramsForExportFile(item, ek, encrypted);
+          var itemParams = new ItemParams(item, ek);
+          return itemParams.paramsForExportFile();
         }.bind(this));
 
         var data = {
           items: items
         }
 
-        if(encrypted && !custom_ek) {
+        if(ek.name == syncManager.SNKeyName) {
           // auth params are only needed when encrypted with a standard file key
           data["auth_params"] = this.getAuthParams();
         }
@@ -481,22 +263,6 @@ angular.module('app.frontend')
         return JSON.parse(JSON.stringify(object));
       }
 
-      this.writeItemsToLocalStorage = function(items, callback) {
-        var params = items.map(function(item) {
-          return this.paramsForItem(item, false, ["created_at", "updated_at", "dirty"], true)
-        }.bind(this));
-
-        dbManager.saveItems(params, callback);
-      }
-
-      this.loadLocalItems = function(callback) {
-        var params = dbManager.getAllItems(function(items){
-          var items = this.handleItemsResponse(items, null, null);
-          Item.sortItemsByDate(items);
-          callback(items);
-        }.bind(this))
-
-      }
 
       /*
       Drafts
@@ -519,101 +285,12 @@ angular.module('app.frontend')
         return modelManager.createItem(jsonObj);
       }
 
-
-      /*
-      Encrpytion
-      */
-
-      this.retrieveMk = function() {
-        if(!this.mk) {
-          this.mk = localStorage.getItem("mk");
-        }
-        return this.mk;
-      }
-
-      this.setMk = function(mk) {
-        localStorage.setItem('mk', mk);
-      }
-
       this.signoutOfStandardFile = function(callback) {
-        this.removeStandardFileSyncProvider();
+        syncManager.removeStandardFileSyncProvider();
         dbManager.clearAllItems(function(){
           localStorage.clear();
           callback();
         });
       }
-
-      this.encryptSingleItem = function(item, masterKey) {
-        var item_key = null;
-        if(item.enc_item_key) {
-          item_key = Neeto.crypto.decryptText(item.enc_item_key, masterKey);
-        } else {
-          item_key = Neeto.crypto.generateRandomEncryptionKey();
-          item.enc_item_key = Neeto.crypto.encryptText(item_key, masterKey);
-        }
-
-        var ek = Neeto.crypto.firstHalfOfKey(item_key);
-        var ak = Neeto.crypto.secondHalfOfKey(item_key);
-        var encryptedContent = "001" + Neeto.crypto.encryptText(JSON.stringify(item.createContentJSONFromProperties()), ek);
-        var authHash = Neeto.crypto.hmac256(encryptedContent, ak);
-
-        item.content = encryptedContent;
-        item.auth_hash = authHash;
-        item.local_encryption_scheme = "1.0";
-      }
-
-       this.decryptSingleItem = function(item, masterKey) {
-         var item_key = Neeto.crypto.decryptText(item.enc_item_key, masterKey);
-
-         var ek = Neeto.crypto.firstHalfOfKey(item_key);
-         var ak = Neeto.crypto.secondHalfOfKey(item_key);
-         var authHash = Neeto.crypto.hmac256(item.content, ak);
-         if(authHash !== item.auth_hash || !item.auth_hash) {
-           console.log("Authentication hash does not match.")
-           return;
-         }
-
-         var content = Neeto.crypto.decryptText(item.content.substring(3, item.content.length), ek);
-         item.content = content;
-       }
-
-       this.decryptItemsWithKey = function(items, key) {
-         for (var item of items) {
-           if(item.deleted == true) {
-             continue;
-           }
-           var isString = typeof item.content === 'string' || item.content instanceof String;
-           if(isString) {
-             try {
-               if(item.content.substring(0, 3) == "001" && item.enc_item_key) {
-                 // is encrypted
-                 this.decryptSingleItem(item, key);
-               } else {
-                 // is base64 encoded
-                 item.content = Neeto.crypto.base64Decode(item.content.substring(3, item.content.length))
-               }
-             } catch (e) {
-               console.log("Error decrypting item", item, e);
-               continue;
-             }
-           }
-         }
-       }
-
-       this.reencryptAllItemsAndSave = function(user, newMasterKey, oldMasterKey, callback) {
-         var items = modelManager.allItems();
-         items.forEach(function(item){
-           if(item.content.substring(0, 3) == "001" && item.enc_item_key) {
-             // first decrypt item_key with old key
-             var item_key = Neeto.crypto.decryptText(item.enc_item_key, oldMasterKey);
-             // now encrypt item_key with new key
-             item.enc_item_key = Neeto.crypto.encryptText(item_key, newMasterKey);
-           }
-         });
-
-         this.saveBatchItems(user, items, function(success) {
-           callback(success);
-         }.bind(this));
-       }
      }
 });

app/assets/javascripts/app/services/directives/accountSyncSection.js

@@ -7,15 +7,15 @@ class AccountSyncSection {
     };
   }
 
-  controller($scope, apiController, modelManager, keyManager) {
+  controller($scope, modelManager, keyManager, syncManager) {
     'ngInject';
 
-      $scope.syncProviders = apiController.syncProviders;
+      $scope.syncProviders = syncManager.syncProviders;
       $scope.newSyncData = {showAddSyncForm: false}
       $scope.keys = keyManager.keys;
 
       $scope.submitExternalSyncURL = function() {
-        apiController.addSyncProviderFromURL($scope.newSyncData.url);
+        syncManager.addSyncProviderFromURL($scope.newSyncData.url);
         $scope.newSyncData.showAddSyncForm = false;
       }
 
@@ -24,11 +24,33 @@ class AccountSyncSection {
           alert("You must choose an encryption key for this provider before enabling it.");
           return;
         }
-        apiController.enableSyncProvider(provider, primary);
+
+        syncManager.enableSyncProvider(provider, primary);
+        syncManager.addAllDataAsNeedingSyncForProvider(provider);
+        syncManager.sync();
       }
 
       $scope.removeSyncProvider = function(provider) {
-        apiController.removeSyncProvider(provider);
+        syncManager.removeSyncProvider(provider);
+      }
+
+      $scope.changeEncryptionKey = function(provider) {
+        if(!confirm("Changing your encryption key will re-encrypt all your notes with the new key and sync them back to the server. This can take several minutes. We strongly recommend downloading a backup of your notes before continuing.")) {
+          return;
+        }
+
+        provider.formData = {keyName: provider.keyName};
+        provider.showKeyForm = true;
+      }
+
+      $scope.saveKey = function(provider) {
+        provider.showKeyForm = false;
+        provider.keyName = provider.formData.keyName;
+
+        if(provider.enabled) {
+          syncManager.addAllDataAsNeedingSyncForProvider(provider);
+          syncManager.sync();
+        }
       }
   }
 }

app/assets/javascripts/app/services/directives/globalExtensionsMenu.js

@@ -7,7 +7,7 @@ class GlobalExtensionsMenu {
     };
   }
 
-  controller($scope, apiController, extensionManager) {
+  controller($scope, extensionManager, syncManager) {
     'ngInject';
 
     $scope.extensionManager = extensionManager;
@@ -39,69 +39,26 @@ class GlobalExtensionsMenu {
           alert("There was an error performing this action. Please try again.");
         } else {
           action.error = false;
-          apiController.sync(null);
+          syncManager.sync(null);
         }
       })
     }
 
     $scope.changeExtensionEncryptionFormat = function(encrypted, extension) {
-      var provider = $scope.syncProviderForExtension(extension);
+      extensionManager.changeExtensionEncryptionFormat(encrypted, extension);
-      if(provider) {
-        if(confirm("Changing encryption status will update all your items and re-sync them back to the server. This can take several minutes. Are you sure you want to continue?")) {
-          extensionManager.changeExtensionEncryptionFormat(encrypted, extension);
-          apiController.resyncAllDataForProvider(provider);
-        } else {
-          // revert setting
-          console.log("reverting");
-          extension.encrypted = extensionManager.extensionUsesEncryptedData(extension);
-        }
-      } else {
-        extensionManager.changeExtensionEncryptionFormat(encrypted, extension);
-      }
     }
 
     $scope.deleteExtension = function(extension) {
       if(confirm("Are you sure you want to delete this extension?")) {
         extensionManager.deleteExtension(extension);
-        var syncProviderAction = extension.syncProviderAction;
-        if(syncProviderAction) {
-          apiController.removeSyncProvider(apiController.syncProviderForURL(syncProviderAction.url));
-        }
       }
     }
 
     $scope.reloadExtensionsPressed = function() {
-      if(confirm("For your security, reloading extensions will disable any currently enabled sync providers and repeat actions.")) {
+      if(confirm("For your security, reloading extensions will disable any currently enabled repeat actions.")) {
         extensionManager.refreshExtensionsFromServer();
-        var syncProviderAction = extension.syncProviderAction;
-        if(syncProviderAction) {
-          apiController.removeSyncProvider(apiController.syncProviderForURL(syncProviderAction.url));
-        }
       }
     }
-
-    $scope.setEncryptionKeyForExtension = function(extension) {
-      extension.formData.changingKey = false;
-      var ek = extension.formData.ek;
-      extensionManager.setEkForExtension(extension, ek);
-      if(extension.formData.changingKey) {
-        var syncAction = extension.syncProviderAction;
-        if(syncAction) {
-          var provider = apiController.syncProviderForURL(syncAction.url);
-          provider.ek = ek;
-          apiController.didMakeChangesToSyncProviders();
-          apiController.resyncAllDataForProvider(provider);
-        }
-      }
-    }
-
-    $scope.changeEncryptionKeyPressed = function(extension) {
-      if(!confirm("Changing your encryption key will re-encrypt all your notes with the new key and sync them back to the server. This can take several minutes. We strongly recommend downloading a backup of your notes before continuing.")) {
-        return;
-      }
-
-      extension.formData.changingKey = true;
-    }
   }
 
 }

app/assets/javascripts/app/services/extensionManager.js

@@ -1,12 +1,13 @@
 class ExtensionManager {
 
-  constructor(Restangular, modelManager, apiController) {
+  constructor(Restangular, modelManager, apiController, syncManager) {
       this.Restangular = Restangular;
       this.modelManager = modelManager;
       this.apiController = apiController;
       this.enabledRepeatActionUrls = JSON.parse(localStorage.getItem("enabledRepeatActionUrls")) || [];
       this.decryptedExtensions = JSON.parse(localStorage.getItem("decryptedExtensions")) || [];
       this.extensionEks = JSON.parse(localStorage.getItem("extensionEks")) || {};
+      this.syncManager = syncManager;
 
       modelManager.addItemSyncObserver("extensionManager", "Extension", function(items){
         for (var ext of items) {
@@ -79,7 +80,7 @@ class ExtensionManager {
     }
 
     this.modelManager.setItemToBeDeleted(extension);
-    this.apiController.sync(null);
+    this.syncManager.sync(null);
   }
 
   /*
@@ -122,7 +123,7 @@ class ExtensionManager {
       extension.url = url;
       extension.setDirty(true);
       this.modelManager.addItem(extension);
-      this.apiController.sync(null);
+      this.syncManager.sync(null);
     }
 
     return extension;
@@ -145,7 +146,8 @@ class ExtensionManager {
 
   executeAction(action, extension, item, callback) {
 
-    if(this.extensionUsesEncryptedData(extension) && !this.apiController.isUserSignedIn()) {
+    //todo
+    if(this.extensionUsesEncryptedData(extension)) {
       alert("To send data encrypted, you must have an encryption key, and must therefore be signed in.");
       callback(null);
       return;
@@ -279,7 +281,8 @@ class ExtensionManager {
   }
 
   outgoingParamsForItem(item, extension) {
-    return this.apiController.paramsForExtension(item, this.extensionUsesEncryptedData(extension));
+    var itemParams = new itemParams(item, extension.ek);
+    return itemParams.paramsForExtension();
   }
 
   performPost(action, extension, params, callback) {

app/assets/javascripts/app/services/sync/encryptionHelper.js

@@ -0,0 +1,63 @@
+class EncryptionHelper {
+
+  encryptItem(item, key) {
+    var item_key = null;
+    if(item.enc_item_key) {
+      item_key = Neeto.crypto.decryptText(item.enc_item_key, key);
+    } else {
+      item_key = Neeto.crypto.generateRandomEncryptionKey();
+      item.enc_item_key = Neeto.crypto.encryptText(item_key, key);
+    }
+
+    var ek = Neeto.crypto.firstHalfOfKey(item_key);
+    var ak = Neeto.crypto.secondHalfOfKey(item_key);
+    var encryptedContent = "001" + Neeto.crypto.encryptText(JSON.stringify(item.createContentJSONFromProperties()), ek);
+    var authHash = Neeto.crypto.hmac256(encryptedContent, ak);
+
+    item.content = encryptedContent;
+    item.auth_hash = authHash;
+    item.local_encryption_scheme = "1.0";
+  }
+
+  decryptItem(item, key) {
+    var item_key = Neeto.crypto.decryptText(item.enc_item_key, key);
+
+    var ek = Neeto.crypto.firstHalfOfKey(item_key);
+    var ak = Neeto.crypto.secondHalfOfKey(item_key);
+    var authHash = Neeto.crypto.hmac256(item.content, ak);
+    if(authHash !== item.auth_hash || !item.auth_hash) {
+      console.log("Authentication hash does not match.")
+      return;
+    }
+
+    var content = Neeto.crypto.decryptText(item.content.substring(3, item.content.length), ek);
+    item.content = content;
+  }
+
+  decryptMultipleItems(items, key) {
+   for (var item of items) {
+     if(item.deleted == true) {
+       continue;
+     }
+
+     var isString = typeof item.content === 'string' || item.content instanceof String;
+     if(isString) {
+       try {
+         if(item.content.substring(0, 3) == "001" && item.enc_item_key) {
+           // is encrypted
+           this.decryptItem(item, key);
+         } else {
+           // is base64 encoded
+           item.content = Neeto.crypto.base64Decode(item.content.substring(3, item.content.length))
+         }
+       } catch (e) {
+         console.log("Error decrypting item", item, e);
+         continue;
+       }
+     }
+    }
+  }
+
+}
+
+angular.module('app.frontend').service('encryptionHelper', EncryptionHelper);

app/assets/javascripts/app/services/sync/syncManager.js

@@ -1,9 +1,26 @@
 class SyncManager {
 
-  let SNKeyName = "Standard Notes Key";
-
   constructor(modelManager) {
     this.modelManager = modelManager;
+    this.syncPerformer = new SyncPerformer()
+    this.SNKeyName = "Standard Notes Key";
+    this.loadSyncProviders();
+  }
+
+  get offline() {
+    return this.syncProviders.length == 0;
+  }
+
+  sync(callback) {
+    this.syncPerformer.sync(this.syncProviders, callback);
+  }
+
+  syncWithProvider(provider, callback) {
+    this.syncPerformer.performSyncWithProvider(provider, callback);
+  }
+
+  loadLocalItems(callback) {
+    this.syncPerformer.loadLocalItems(callback);
   }
 
   syncProviderForURL(url) {
@@ -36,7 +53,7 @@ class SyncManager {
 
   addStandardFileSyncProvider(url) {
     var defaultProvider = new SyncProvider({url: url + "/items/sync", primary: this.syncProviders.length == 0});
-    defaultProvider.keyName = SNKeyName;
+    defaultProvider.keyName = this.SNKeyName;
     defaultProvider.enabled = this.syncProviders.length == 0;
     this.syncProviders.push(defaultProvider);
     return defaultProvider;
@@ -64,7 +81,7 @@ class SyncManager {
         // migrate old key structure to new
         var mk = localStorage.getItem("mk");
         if(mk) {
-          keyManager.addKey(SNKeyName, mk);
+          keyManager.addKey(this.SNKeyName, mk);
           localStorage.removeItem("mk");
         }
         this.didMakeChangesToSyncProviders();
@@ -72,8 +89,6 @@ class SyncManager {
     }
   }
 
-  this.loadSyncProviders();
-
   addSyncProviderFromURL(url) {
     var provider = new SyncProvider({url: url});
     this.syncProviders.push(provider);
@@ -91,13 +106,13 @@ class SyncManager {
     syncProvider.primary = primary;
 
     // since we're enabling a new provider, we need to send it EVERYTHING we have now.
-    syncProvider.addPendingItems(this.modelManager.allItems);
+    this.addAllDataAsNeedingSyncForProvider(syncProvider);
     this.didMakeChangesToSyncProviders();
+    this.syncWithProvider(syncProvider);
   }
 
-  resyncAllDataForProvider(syncProvider) {
+  addAllDataAsNeedingSyncForProvider(syncProvider) {
     syncProvider.addPendingItems(this.modelManager.allItems);
-    this.sync();
   }
 
   removeSyncProvider(provider) {
@@ -105,7 +120,52 @@ class SyncManager {
     this.didMakeChangesToSyncProviders();
   }
 
-
+  clearSyncToken() {
+    var primary = this.primarySyncProvider();
+    if(primary) {
+      primary.syncToken = null;
+    }
+  }
 }
 
 angular.module('app.frontend').service('syncManager', SyncManager);
+
+class SyncProvider {
+
+  constructor(obj) {
+    this.encrypted = true;
+    _.merge(this, obj);
+  }
+
+  addPendingItems(items) {
+    if(!this.pendingItems) {
+      this.pendingItems = [];
+    }
+
+    this.pendingItems = this.pendingItems.concat(items);
+  }
+
+  removePendingItems(items) {
+    this.pendingItems = _.difference(this.pendingItems, items);
+  }
+
+  get status() {
+    if(!this.enabled) {
+      return null;
+    }
+
+    if(this.primary) return "primary";
+    else return "secondary";
+  }
+
+  asJSON() {
+    return {
+      enabled: this.enabled,
+      url: this.url,
+      primary: this.primary,
+      keyName: this.keyName,
+      syncToken: this.syncToken
+    }
+  }
+
+}

app/assets/javascripts/app/services/sync/syncPerformer.js

@@ -0,0 +1,196 @@
+class SyncPerformer {
+
+  constructor(modelManager, dbManager, encryptionHelper, keyManager) {
+    this.modelManager = modelManager;
+    this.dbManager = dbManager;
+    this.encryptionHelper = encryptionHelper;
+    this.keyManager = keyManager;
+  }
+
+  setOnChangeProviderCallback(callback) {
+    this.onChangeProviderCallback = callback;
+  }
+
+  didMakeChangesToSyncProvider(provider) {
+    this.onChangeProviderCallback(provider);
+  }
+
+  writeItemsToLocalStorage(items, callback) {
+    var params = items.map(function(item) {
+      return this.paramsForItem(item, false, ["created_at", "updated_at", "dirty"], true)
+    }.bind(this));
+
+    this.dbManager.saveItems(params, callback);
+  }
+
+  loadLocalItems(callback) {
+    var params = this.dbManager.getAllItems(function(items){
+      var items = this.handleItemsResponse(items, null, null);
+      Item.sortItemsByDate(items);
+      callback(items);
+    }.bind(this))
+  }
+
+  syncOffline(items, callback) {
+    this.writeItemsToLocalStorage(items, function(responseItems){
+      // delete anything needing to be deleted
+      for(var item of items) {
+        if(item.deleted) {
+          this.modelManager.removeItemLocally(item);
+        }
+      }
+    }.bind(this))
+
+    if(callback) {
+      callback();
+    }
+  }
+
+  sync(providers, callback, options = {}) {
+
+    var allDirtyItems = this.modelManager.getDirtyItems();
+
+    // we want to write all dirty items to disk only if the user has no sync providers, or if the sync op fails
+    // if the sync op succeeds, these items will be written to disk by handling the "saved_items" response from the server
+    if(providers.length == 0) {
+      this.syncOffline(allDirtyItems, callback);
+    }
+
+    for(let provider of providers) {
+      if(!provider.enabled) {
+        continue;
+      }
+
+      provider.addPendingItems(allDirtyItems);
+      this.didMakeChangesToSyncProvider(provider);
+
+      this.__performSyncWithProvider(provider, options, function(response){
+        if(provider.primary) {
+          if(callback) {
+            callback(response)
+          }
+        }
+      })
+    }
+
+    this.modelManager.clearDirtyItems(allDirtyItems);
+  }
+
+  performSyncWithProvider(provider, callback) {
+    this.__performSyncWithProvider(provider, {}, callback);
+  }
+
+  __performSyncWithProvider(provider, options, callback) {
+    if(provider.syncOpInProgress) {
+      provider.repeatOnCompletion = true;
+      console.log("Sync op in progress for provider; returning.", provider);
+      return;
+    }
+
+
+    provider.syncOpInProgress = true;
+
+    let submitLimit = 100;
+    var allItems = provider.pendingItems;
+    var subItems = allItems.slice(0, submitLimit);
+    if(subItems.length < allItems.length) {
+      // more items left to be synced, repeat
+      provider.repeatOnCompletion = true;
+    } else {
+      provider.repeatOnCompletion = false;
+    }
+
+    console.log("Syncing with provider", provider, subItems);
+
+    // Remove dirty items now. If this operation fails, we'll re-add them.
+    // This allows us to queue changes on the same item
+    provider.removePendingItems(subItems);
+
+    var request = Restangular.oneUrl(provider.url, provider.url);
+    request.limit = 150;
+    request.items = _.map(subItems, function(item){
+      var itemParams = new ItemParams(item, provider.ek);
+      itemParams.additionalFields = options.additionalFields;
+      return itemParams.paramsForSync();
+    }.bind(this));
+
+    request.sync_token = provider.syncToken;
+    request.cursor_token = provider.cursorToken;
+
+    request.post().then(function(response) {
+
+      console.log("Sync completion", response);
+
+      provider.syncToken = response.sync_token;
+
+      if(provider.primary) {
+        $rootScope.$broadcast("sync:updated_token", provider.syncToken);
+
+        // handle cursor token (more results waiting, perform another sync)
+        provider.cursorToken = response.cursor_token;
+
+        var retrieved = this.handleItemsResponse(response.retrieved_items, null, provider);
+        // merge only metadata for saved items
+        var omitFields = ["content", "auth_hash"];
+        var saved = this.handleItemsResponse(response.saved_items, omitFields, provider);
+
+        this.handleUnsavedItemsResponse(response.unsaved, provider)
+
+        this.writeItemsToLocalStorage(saved, null);
+        this.writeItemsToLocalStorage(retrieved, null);
+      }
+
+      provider.syncOpInProgress = false;
+
+      if(provider.cursorToken || provider.repeatOnCompletion == true) {
+        this.__performSyncWithProvider(provider, options, callback);
+      } else {
+        if(callback) {
+          callback(response);
+        }
+      }
+
+    }.bind(this))
+    .catch(function(response){
+      console.log("Sync error: ", response);
+
+      // Re-add subItems since this operation failed. We'll have to try again.
+      provider.addPendingItems(subItems);
+      provider.syncOpInProgress = false;
+
+      if(provider.primary) {
+        this.writeItemsToLocalStorage(allItems, null);
+      }
+
+      if(callback) {
+        callback({error: "Sync error"});
+      }
+    }.bind(this))
+  }
+
+  handleUnsavedItemsResponse(unsaved, provider) {
+    if(unsaved.length == 0) {
+      return;
+    }
+
+    console.log("Handle unsaved", unsaved);
+    for(var mapping of unsaved) {
+      var itemResponse = mapping.item;
+      var item = this.modelManager.findItem(itemResponse.uuid);
+      var error = mapping.error;
+      if(error.tag == "uuid_conflict") {
+          item.alternateUUID();
+          item.setDirty(true);
+          item.markAllReferencesDirty();
+      }
+    }
+
+    this.__performSyncWithProvider(provider, {additionalFields: ["created_at", "updated_at"]}, null);
+  }
+
+  handleItemsResponse(responseItems, omitFields, syncProvider) {
+    var ek = syncProvider ? this.keyManager.keyForName(syncProvider.keyName).key : null;
+    this.encryptionHelper.decryptMultipleItems(responseItems, ek);
+    return this.modelManager.mapResponseItemsToLocalModelsOmittingFields(responseItems, omitFields);
+  }
+}

app/assets/templates/frontend/directives/account-sync-section.html.haml

@@ -5,14 +5,16 @@
     .url {{provider.url}}
     .options
       %div{"ng-if" => "!provider.enabled"}
-        %strong Choose encryption key:
+        %div{"ng-if" => "!provider.keyName || provider.showKeyForm"}
-        %select{"ng-model" => "provider.keyName"}
+          %strong Choose encryption key:
-          %option{"ng-repeat" => "key in keys", "ng-selected" => "{{key.name == provider.keyName}}", "value" => "{{key.name}}"}
+          %select{"ng-model" => "provider.formData.keyName"}
-            {{key.name}}
+            %option{"ng-repeat" => "key in keys", "ng-selected" => "{{key.name == provider.formData.keyName}}", "value" => "{{key.name}}"}
+              {{key.name}}
+          %button{"ng-click" => "saveKey(provider)"} Set
         %button.light{"ng-click" => "enableSyncProvider(provider, true)"} Enable as Primary sync provider
         %button.light{"ng-click" => "enableSyncProvider(provider, false)"} Enable as Secondary sync provider
-      %button.light{"ng-click" => "removeSyncProvider(provider)"} Remove Provider
       %button.light{"ng-if" => "provider.keyName", "ng-click" => "changeEncryptionKey(provider)"} Change Encryption Key
+      %button.light{"ng-click" => "removeSyncProvider(provider)"} Remove Provider
 
   %a{"ng-click" => "newSyncData.showAddSyncForm = !newSyncData.showAddSyncForm"} Add external sync with Secret URL
   %form.sync-form{"ng-if" => "newSyncData.showAddSyncForm"}

app/assets/templates/frontend/directives/global-extensions-menu.html.haml

@@ -12,38 +12,22 @@
           %label
             %input{"type" => "radio", "ng-model" => "extension.encrypted", "ng-value" => "false", "ng-change" => "changeExtensionEncryptionFormat(false, extension)"}
             Decrypted
-        .ek-input-wrapper{"ng-if" => "extension.encrypted && (!extensionManager.ekForExtension(extension) || extension.formData.changingKey)"}
-          %input{"ng-model" => "extension.formData.ek", "placeholder" => "Set encryption key"}
-          %button.light{"ng-click" => "setEncryptionKeyForExtension(extension)"} Set
         .extension-actions
           .action{"ng-repeat" => "action in extension.actionsInGlobalContext()"}
-            %div{"ng-if" => "!action.sync_provider"}
+            .action-name {{action.label}}
-              .action-name {{action.label}}
+            .action-desc{"style" => "font-style: italic;"} {{action.desc}}
-              .action-desc{"style" => "font-style: italic;"} {{action.desc}}
+            .execute-type{"ng-if" => "action.repeat_mode == 'watch'"}
-              .execute-type{"ng-if" => "action.repeat_mode == 'watch'"}
+              Repeats when a change is made to your items.
-                Repeats when a change is made to your items.
+            .execute-type{"ng-if" => "action.repeat_mode == 'loop'"}
-              .execute-type{"ng-if" => "action.repeat_mode == 'loop'"}
+              Repeats at most once every {{action.repeat_timeout}} seconds
-                Repeats at most once every {{action.repeat_timeout}} seconds
+            .action-permissions
-              .action-permissions
+              %a{"ng-click" => "action.showPermissions = !action.showPermissions"} {{action.showPermissions ? "Hide permissions" : "Show permissions"}}
-                %a{"ng-click" => "action.showPermissions = !action.showPermissions"} {{action.showPermissions ? "Hide permissions" : "Show permissions"}}
+              %div{"ng-if" => "action.showPermissions"}
-                %div{"ng-if" => "action.showPermissions"}
+                {{action.permissionsString}}
-                  {{action.permissionsString}}
+                .encryption-type
-                  .encryption-type
+                  %span {{action.encryptionModeString}}
-                    %span {{action.encryptionModeString}}
-            %div{"ng-if" => "action.sync_provider"}
-              .action-name This is a sync provider action.
-              .action-desc{"style" => "margin-top: -5px;"}
-                %p Enabling this sync provider as a primary provider will replace your current sync provider.
-                %p Enabling this sync provider as a secondary provider will save your data with this provider as a backup, but will not be used to pull changes.
-                %p You can have only one primary provider, and multiple secondary providers.
 
-              %div{"ng-if" => "!syncProviderActionIsEnabled(action)"}
+            .execute
-                %button.light.execute{"ng-click" => "enableSyncProvider(action, extension, true)"} Enable as primary sync provider
-                %button.light.execute{"ng-click" => "enableSyncProvider(action, extension, false)"} Enable as backup sync provider
-              %div{"ng-if" => "syncProviderActionIsEnabled(action)"}
-                %button.light.execute{"ng-click" => "disableSyncProvider(action, extension)"} Remove as sync provider
-
-            .execute{"ng-if" => "!action.sync_provider"}
               %div{"ng-if" => "action.repeat_mode"}
                 %div{"ng-if" => "extensionManager.isRepeatActionEnabled(action)", "ng-click" => "extensionManager.disableRepeatAction(action, extension)"} Disable
                 %div{"ng-if" => "!extensionManager.isRepeatActionEnabled(action)", "ng-click" => "extensionManager.enableRepeatAction(action, extension)"} Enable
@@ -57,14 +41,6 @@
             .error{"ng-if" => "action.error"}
               Error performing action.
 
-
-        %a.option-link{"ng-if" => "extension.encrypted && extensionManager.ekForExtension(extension) && !extension.formData.changingKey", "ng-click" => "extension.formData.showEk = !extension.formData.showEk"}
-          Show Encryption Key
-        .show-ek{"style" => "text-align: center", "ng-if" => "extension.formData.showEk"}
-          .ek {{extensionManager.ekForExtension(extension)}}
-          .disclaimer This key is saved locally and never sent to any servers.
-        %a.option-link{"ng-if" => "extension.encrypted && extensionManager.ekForExtension(extension) && !extension.formData.changingKey", "ng-click" => "changeEncryptionKeyPressed(extension)"}
-          Change Encryption Key
         %a.option-link{"ng-click" => "deleteExtension(extension)"} Remove extension
 
     .extension-link