sync providers wip

app/assets/javascripts/app/frontend/controllers/header.js

@@ -71,15 +71,47 @@ angular.module('app.frontend')
       })
     }
 
+    this.syncProviderActionIsEnabled = function(action) {
+      var provider = apiController.syncProviderForURL(action.url);
+      if(!provider) {
+        return null;
+      }
+      return provider.status;
+    }
+
+    this.enableSyncProvider = function(action, extension, primary) {
+      if(extension.encrypted && !extension.ek) {
+        alert("You must set an encryption key for this extension before enabling this action.");
+        return;
+      }
+      var provider = apiController.findOrCreateSyncProviderForUrl(action.url);
+      provider.primary = primary;
+      provider.enabled = true;
+      provider.ek = extension.ek;
+      apiController.addSyncProvider(provider);
+    }
+
+    this.disableSyncProvider = function(action, extension) {
+      apiController.removeSyncProvider(apiController.syncProviderForURL(action.url));
+    }
+
     this.deleteExtension = function(extension) {
       if(confirm("Are you sure you want to delete this extension?")) {
         extensionManager.deleteExtension(extension);
+        var syncProviderAction = extension.syncProviderAction;
+        if(syncProviderAction) {
+          apiController.removeSyncProvider(apiController.syncProviderForURL(syncProviderAction.url));
+        }
       }
     }
 
     this.reloadExtensionsPressed = function() {
-      if(confirm("For your security, reloading extensions will disable any currently enabled repeat actions.")) {
+      if(confirm("For your security, reloading extensions will disable any currently enabled sync providers and repeat actions.")) {
         extensionManager.refreshExtensionsFromServer();
+        var syncProviderAction = extension.syncProviderAction;
+        if(syncProviderAction) {
+          apiController.removeSyncProvider(apiController.syncProviderForURL(syncProviderAction.url));
+        }
       }
     }
 

app/assets/javascripts/app/frontend/models/app/extension.js

@@ -56,9 +56,13 @@ class Extension extends Item {
       this.content_type = "Extension";
   }
 
+  syncProviderAction() {
+    return _.find(this.actions, {sync_provider: true})
+  }
+
   actionsInGlobalContext() {
     return this.actions.filter(function(action){
-      return action.context == "global";
+      return action.context == "global" || action.sync_provider == true;
     })
   }
 

app/assets/javascripts/app/frontend/models/sync/syncProvider.js

@@ -0,0 +1,36 @@
+class SyncProvider {
+  constructor(obj) {
+    _.merge(this, obj);
+  }
+
+  addPendingItems(items) {
+    if(!this.pendingItems) {
+      this.pendingItems = [];
+    }
+
+    this.pendingItems = this.pendingItems.concat(items);
+  }
+
+  removePendingItems(items) {
+    this.pendingItems = _.difference(this.pendingItems, items);
+  }
+
+  get status() {
+    if(!this.enabled) {
+      return null;
+    }
+
+    if(this.primary) return "primary";
+    else return "secondary";
+  }
+
+  asJSON() {
+    return {
+      enabled: this.enabled,
+      url: this.url,
+      encrypted: this.encrypted,
+      ek: this.ek
+    }
+  }
+
+}

app/assets/javascripts/app/services/apiController.js

@@ -201,9 +201,71 @@ angular.module('app.frontend')
 
 
       /*
-      Items
+      Sync
       */
 
+      this.syncProviderForURL = function(url) {
+        return _.find(this.syncProviders, {url: url});
+      }
+
+      this.findOrCreateSyncProviderForUrl = function(url) {
+        var provider = _.find(this.syncProviders, {url: url});
+        if(!provider) {
+          provider = new SyncProvider({url: url})
+        }
+        return provider;
+      }
+
+      this.setEncryptionStatusForProviderURL = function(providerURL, encrypted) {
+        this.providerForURL(providerURL).encrypted = encrypted;
+        this.persistSyncProviders();
+      }
+
+      this.loadSyncProviders = function() {
+        var providers = [];
+        var saved = localStorage.getItem("syncProviders");
+        if(saved) {
+          var parsed = JSON.parse(saved);
+          for(var p of parsed) {
+            providers.push(new SyncProvider(p));
+          }
+        } else {
+          // no providers saved, use default
+          if(this.isUserSignedIn()) {
+            var defaultProvider = new SyncProvider(this.getServer() + "/items/sync", true);
+            providers.push(defaultProvider);
+          }
+        }
+
+        this.syncProviders = providers;
+      }
+      this.loadSyncProviders();
+
+      this.addSyncProvider = function(syncProvider) {
+        if(syncProvider.primary) {
+          for(var provider of this.syncProviders) {
+            provider.primary = false;
+          }
+        }
+
+        // since we're adding a new provider, we need to send it EVERYTHING we have now.
+        syncProvider.addPendingItems(modelManager.allItems);
+
+        this.syncProviders.push(syncProvider);
+        this.persistSyncProviders();
+      }
+
+      this.removeSyncProvider = function(provider) {
+        _.pull(this.syncProviders, provider);
+        this.persistSyncProviders();
+      }
+
+      this.persistSyncProviders = function() {
+        localStorage.setItem("syncProviders", JSON.stringify(_.map(this.syncProviders, function(provider) {
+          return provider.asJSON()
+        })));
+      }
+
       this.setSyncToken = function(syncToken) {
         this.syncToken = syncToken;
         localStorage.setItem("syncToken", this.syncToken);
@@ -211,14 +273,6 @@ angular.module('app.frontend')
 
       this.syncWithOptions = function(callback, options = {}) {
 
-        if(this.syncOpInProgress) {
-          // will perform anoter sync after current completes
-          this.repeatSync = true;
-          return;
-        }
-
-        this.syncOpInProgress = true;
-
         var allDirtyItems = modelManager.getDirtyItems();
 
         // we want to write all dirty items to disk only if the user is not signed in, or if the sync op fails
@@ -243,55 +297,94 @@ angular.module('app.frontend')
             modelManager.clearDirtyItems(allDirtyItems);
 
           }.bind(this))
+
           this.syncOpInProgress = false;
-          if(callback) {
+
+          if(this.syncProviders.length == 0 && callback) {
             callback();
           }
+        }
+
+        for(let provider of this.syncProviders) {
+          if(provider.enabled == false) {
+            continue;
+          }
+          provider.addPendingItems(allDirtyItems);
+          this.__performSyncWithProvider(provider, options, function(response){
+            if(provider.primary) {
+              if(callback) {
+                callback(response)
+              }
+            }
+          })
+        }
+
+        modelManager.clearDirtyItems(allDirtyItems);
+      }
+
+      this.__performSyncWithProvider = function(provider, options, callback) {
+        if(provider.syncOpInProgress) {
+          provider.repeatOnCompletion = true;
+          console.log("Sync op in progress for provider; returning.", provider);
           return;
         }
 
+
+        provider.syncOpInProgress = true;
+
         let submitLimit = 100;
-        var dirtyItems = allDirtyItems.slice(0, submitLimit);
+        var allItems = provider.pendingItems;
-        if(dirtyItems.length < allDirtyItems.length) {
+        var subItems = allItems.slice(0, submitLimit);
+        if(subItems.length < allItems.length) {
           // more items left to be synced, repeat
-          this.repeatSync = true;
+          provider.repeatOnCompletion = true;
         } else {
-          this.repeatSync = false;
+          provider.repeatOnCompletion = false;
         }
 
-        var request = Restangular.one("items/sync");
+        console.log("Syncing with provider", provider, subItems);
+
+        // Remove dirty items now. If this operation fails, we'll re-add them.
+        // This allows us to queue changes on the same item
+        provider.removePendingItems(subItems);
+
+        var request = Restangular.oneUrl(provider.url, provider.url);
         request.limit = 150;
-        request.sync_token = this.syncToken;
+        request.items = _.map(subItems, function(item){
-        request.cursor_token = this.cursorToken;
+          return this.paramsForItem(item, provider.encrypted, provider.ek, options.additionalFields, false);
-        request.items = _.map(dirtyItems, function(item){
-          return this.createRequestParamsForItem(item, options.additionalFields);
         }.bind(this));
 
+        if(provider.primary) {
+          // only primary providers receive items (or care about received items)
+          request.sync_token = this.syncToken;
+          request.cursor_token = provider.cursorToken;
+        }
+
         request.post().then(function(response) {
 
-          modelManager.clearDirtyItems(dirtyItems);
+          if(provider.primary) {
+            // handle sync token
+            this.setSyncToken(response.sync_token);
+            $rootScope.$broadcast("sync:updated_token", this.syncToken);
 
-          // handle sync token
+            // handle cursor token (more results waiting, perform another sync)
-          this.setSyncToken(response.sync_token);
+            provider.cursorToken = response.cursor_token;
-          $rootScope.$broadcast("sync:updated_token", this.syncToken);
 
-          // handle cursor token (more results waiting, perform another sync)
+            var retrieved = this.handleItemsResponse(response.retrieved_items, null, provider);
-          this.cursorToken = response.cursor_token;
+            // merge only metadata for saved items
+            var omitFields = ["content", "auth_hash"];
+            var saved = this.handleItemsResponse(response.saved_items, omitFields, provider);
 
-          var retrieved = this.handleItemsResponse(response.retrieved_items, null);
+            this.handleUnsavedItemsResponse(response.unsaved, provider)
-          // merge only metadata for saved items
-          var omitFields = ["content", "auth_hash"];
-          var saved = this.handleItemsResponse(response.saved_items, omitFields);
 
-          this.handleUnsavedItemsResponse(response.unsaved)
+            this.writeItemsToLocalStorage(saved, null);
+            this.writeItemsToLocalStorage(retrieved, null);
+          }
 
-          this.writeItemsToLocalStorage(saved, null);
+          provider.syncOpInProgress = false;
-          this.writeItemsToLocalStorage(retrieved, null);
 
-          this.syncOpInProgress = false;
+          if(provider.cursorToken || provider.repeatOnCompletion == true) {
-
+            this.__performSyncWithProvider(provider, options, callback);
-          if(this.cursorToken || this.repeatSync == true) {
-            this.syncWithOptions(callback, options);
           } else {
             if(callback) {
               callback(response);
@@ -302,8 +395,13 @@ angular.module('app.frontend')
         .catch(function(response){
           console.log("Sync error: ", response);
 
-          writeAllDirtyItemsToDisk();
+          // Re-add subItems since this operation failed. We'll have to try again.
-          this.syncOpInProgress = false;
+          provider.addPendingItems(subItems);
+          provider.syncOpInProgress = false;
+
+          if(provider.primary) {
+            this.writeItemsToLocalStorage(allItems, null);
+          }
 
           if(callback) {
             callback({error: "Sync error"});
@@ -315,7 +413,7 @@ angular.module('app.frontend')
         this.syncWithOptions(callback, undefined);
       }
 
-      this.handleUnsavedItemsResponse = function(unsaved) {
+      this.handleUnsavedItemsResponse = function(unsaved, provider) {
         if(unsaved.length == 0) {
           return;
         }
@@ -332,19 +430,15 @@ angular.module('app.frontend')
           }
         }
 
-        this.syncWithOptions(null, {additionalFields: ["created_at", "updated_at"]});
+        this.__performSyncWithProvider(provider, {additionalFields: ["created_at", "updated_at"]}, null);
       }
 
-      this.handleItemsResponse = function(responseItems, omitFields) {
+      this.handleItemsResponse = function(responseItems, omitFields, syncProvider) {
-        this.decryptItems(responseItems);
+        this.decryptItemsWithKey(responseItems, syncProvider ? syncProvider.ek : null);
         return modelManager.mapResponseItemsToLocalModelsOmittingFields(responseItems, omitFields);
       }
 
-      this.createRequestParamsForItem = function(item, additionalFields) {
+      this.paramsForExportFile = function(item, ek, encrypted) {
-        return this.paramsForItem(item, true, additionalFields, false);
-      }
-
-      this.paramsForExportFile = function(item, encrypted) {
         return _.omit(this.paramsForItem(item, encrypted, ["created_at", "updated_at"], true), ["deleted"]);
       }
 
@@ -352,15 +446,18 @@ angular.module('app.frontend')
         return _.omit(this.paramsForItem(item, encrypted, ["created_at", "updated_at"], true), ["deleted"]);
       }
 
-      this.paramsForItem = function(item, encrypted, additionalFields, forExportFile) {
+      this.paramsForItem = function(item, encrypted, ek, additionalFields, forExportFile) {
         var itemCopy = _.cloneDeep(item);
 
+        if(encrypted) {
+          console.assert(ek.length, "Attempting to encrypt without encryption key.");
+        }
         console.assert(!item.dummy, "Item is dummy, should not have gotten here.", item.dummy)
 
         var params = {uuid: item.uuid, content_type: item.content_type, deleted: item.deleted};
 
         if(encrypted) {
-          this.encryptSingleItem(itemCopy, this.retrieveMk());
+          this.encryptSingleItem(itemCopy, ek);
           params.content = itemCopy.content;
           params.enc_item_key = itemCopy.enc_item_key;
           params.auth_hash = itemCopy.auth_hash;
@@ -475,7 +572,7 @@ angular.module('app.frontend')
 
       this.loadLocalItems = function(callback) {
         var params = dbManager.getAllItems(function(items){
-          var items = this.handleItemsResponse(items, null);
+          var items = this.handleItemsResponse(items, null, null);
           Item.sortItemsByDate(items);
           callback(items);
         }.bind(this))
@@ -560,11 +657,6 @@ angular.module('app.frontend')
          item.content = content;
        }
 
-       this.decryptItems = function(items) {
-         var masterKey = this.retrieveMk();
-         this.decryptItemsWithKey(items, masterKey);
-       }
-
        this.decryptItemsWithKey = function(items, key) {
          for (var item of items) {
            if(item.deleted == true) {

app/assets/stylesheets/app/_header.scss

@@ -312,7 +312,7 @@ Extensions
       font-weight: bold !important;
     }
 
-    > .name {
+    .extension-name {
       font-weight: bold;
       font-size: 16px;
       margin-bottom: 6px;
@@ -331,12 +331,12 @@ Extensions
       }
     }
 
-    > .subtitle {
+    .extension-subtitle {
       font-size: 14px;
       margin-bottom: 10px;
     }
 
-    > .actions {
+    .extension-actions {
       margin-top: 15px;
       font-size: 12px;
 
@@ -346,18 +346,18 @@ Extensions
         background-color: rgba(white, 0.9);
         border: 1px solid rgba(gray, 0.15);
 
-        > .name {
+       .action-name {
           font-weight: bold;
         }
 
-        > .permissions {
+        .action-permissions {
           margin-top: 2px;
           a {
             font-weight: normal !important;
           }
         }
 
-        > .execute {
+       .execute {
           font-weight: bold;
           margin-bottom: 0px;
           font-size: 12px;

app/assets/templates/frontend/header.html.haml

@@ -103,7 +103,7 @@
               %div{"style" => "font-size: 18px;", "ng-if" => "!ctrl.extensionManager.extensions.length"} No extensions installed
               .registered-extensions{"ng-if" => "ctrl.extensionManager.extensions.length"}
                 .extension{"ng-repeat" => "extension in ctrl.extensionManager.extensions"}
-                  .name {{extension.name}}
+                  .extension-name {{extension.name}}
                   .encryption-format
                     .title Send data:
                     %label
@@ -112,21 +112,37 @@
                     %label
                       %input{"type" => "radio", "ng-model" => "extension.encrypted", "ng-value" => "false", "ng-change" => "ctrl.extensionManager.changeExtensionEncryptionFormat(false, extension)"}
                       Decrypted
-                  .actions
+                  .encryption-key{"ng-if" => "extension.encrypted"}
+                    %input{"ng-model" => "extension.ek", "placeholder" => "Set encryption key"}
+                  .extension-actions
                     .action{"ng-repeat" => "action in extension.actionsInGlobalContext()"}
-                      .name {{action.label}}
+                      %div{"ng-if" => "!action.sync_provider"}
-                      .desc{"style" => "font-style: italic;"} {{action.desc}}
+                        .action-name {{action.label}}
-                      .execute-type{"ng-if" => "action.repeat_mode == 'watch'"}
+                        .action-desc{"style" => "font-style: italic;"} {{action.desc}}
-                        Repeats when a change is made to your items.
+                        .execute-type{"ng-if" => "action.repeat_mode == 'watch'"}
-                      .execute-type{"ng-if" => "action.repeat_mode == 'loop'"}
+                          Repeats when a change is made to your items.
-                        Repeats at most once every {{action.repeat_timeout}} seconds
+                        .execute-type{"ng-if" => "action.repeat_mode == 'loop'"}
-                      .permissions
+                          Repeats at most once every {{action.repeat_timeout}} seconds
-                        %a{"ng-click" => "action.showPermissions = !action.showPermissions"} {{action.showPermissions ? "Hide permissions" : "Show permissions"}}
+                        .action-permissions
-                        %div{"ng-if" => "action.showPermissions"}
+                          %a{"ng-click" => "action.showPermissions = !action.showPermissions"} {{action.showPermissions ? "Hide permissions" : "Show permissions"}}
-                          {{action.permissionsString}}
+                          %div{"ng-if" => "action.showPermissions"}
-                          .encryption-type
+                            {{action.permissionsString}}
-                            %span {{action.encryptionModeString}}
+                            .encryption-type
-                      .execute
+                              %span {{action.encryptionModeString}}
+                      %div{"ng-if" => "action.sync_provider"}
+                        .action-name This is a sync provider action.
+                        .action-desc{"style" => "margin-top: -5px;"}
+                          %p Enabling this sync provider as a primary provider will replace your current sync provider.
+                          %p Enabling this sync provider as a secondary provider will save your data with this provider as a backup, but will not be used to pull changes.
+                          %p You can have only one primary provider, and multiple secondary providers.
+
+                        %div{"ng-if" => "!ctrl.syncProviderActionIsEnabled(action)"}
+                          .execute{"ng-click" => "ctrl.enableSyncProvider(action, extension, true)"} Enable as primary sync provider
+                          .execute{"ng-click" => "ctrl.enableSyncProvider(action, extension, false)"} Enable as backup sync provider
+                        %div{"ng-if" => "ctrl.syncProviderActionIsEnabled(action)"}
+                          .execute{"ng-click" => "ctrl.disableSyncProvider(action, extension)"} Remove as sync provider
+
+                      .execute{"ng-if" => "!action.sync_provider"}
                         %div{"ng-if" => "action.repeat_mode"}
                           %div{"ng-if" => "ctrl.extensionManager.isRepeatActionEnabled(action)", "ng-click" => "ctrl.extensionManager.disableRepeatAction(action, extension)"} Disable
                           %div{"ng-if" => "!ctrl.extensionManager.isRepeatActionEnabled(action)", "ng-click" => "ctrl.extensionManager.enableRepeatAction(action, extension)"} Enable