feat: Moments: your personal photo journal, now available in Labs (#2079)

packages/web/src/javascripts/Controllers/LinkingController.tsx

@@ -240,7 +240,7 @@ export class LinkingController extends AbstractViewController {
     }
   }
 
-  linkItems = async (item: LinkableItem, itemToLink: LinkableItem) => {
+  linkItems = async (item: SNNote | FileItem, itemToLink: LinkableItem) => {
     if (item instanceof SNNote) {
       if (itemToLink instanceof FileItem) {
         await this.application.items.associateFileWithNote(itemToLink, item)
@@ -248,6 +248,8 @@ export class LinkingController extends AbstractViewController {
         await this.application.items.linkNoteToNote(item, itemToLink)
       } else if (itemToLink instanceof SNTag) {
         await this.addTagToItem(itemToLink, item)
+      } else {
+        throw Error('Invalid item type')
       }
     } else if (item instanceof FileItem) {
       if (itemToLink instanceof SNNote) {
@@ -256,7 +258,11 @@ export class LinkingController extends AbstractViewController {
         await this.application.items.linkFileToFile(item, itemToLink)
       } else if (itemToLink instanceof SNTag) {
         await this.addTagToItem(itemToLink, item)
+      } else {
+        throw Error('Invalid item to link')
       }
+    } else {
+      throw new Error('First item must be a note or file')
     }
 
     void this.application.sync.sync()

packages/web/src/javascripts/Controllers/Moments/CameraUtils.ts

@@ -0,0 +1,65 @@
+export async function awaitVideoReady(video: HTMLVideoElement) {
+  return new Promise((resolve) => {
+    video.addEventListener('canplaythrough', () => {
+      resolve(null)
+    })
+  })
+}
+
+export function stopCameraStream(canvas: HTMLCanvasElement, video: HTMLVideoElement, stream: MediaStream) {
+  video.pause()
+  video.parentElement?.removeChild(video)
+  canvas.parentElement?.removeChild(canvas)
+  video.remove()
+  canvas.remove()
+
+  stream.getTracks().forEach((track) => {
+    track.stop()
+  })
+}
+
+export async function takePhoto(
+  filename: string,
+  canvas: HTMLCanvasElement,
+  video: HTMLVideoElement,
+  width: number,
+  height: number,
+): Promise<File | undefined> {
+  const context = canvas.getContext('2d')
+  context?.drawImage(video, 0, 0, width, height)
+  const dataUrl = canvas.toDataURL('image/png')
+
+  const isFailedImage = dataUrl.length < 100000
+
+  if (isFailedImage) {
+    return undefined
+  }
+
+  const res: Response = await fetch(dataUrl)
+  const blob: Blob = await res.blob()
+  const file = new File([blob], filename, { type: 'image/png' })
+  return file
+}
+
+export async function preparePhotoOperation() {
+  const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false })
+  const video = document.createElement('video')
+  video.playsInline = true
+  video.style.position = 'absolute'
+  video.style.display = 'none'
+  const canvas = document.createElement('canvas')
+  document.body.append(video)
+  video.srcObject = stream
+  await video.play()
+  await awaitVideoReady(video)
+
+  const videoTrack = stream.getVideoTracks()[0]
+  const settings = videoTrack.getSettings()
+  const width = settings.width ?? 1280
+  const height = settings.height ?? 720
+
+  canvas.width = width
+  canvas.height = height
+
+  return { canvas, video, stream, width, height }
+}

packages/web/src/javascripts/Controllers/Moments/MomentsService.ts

@@ -0,0 +1,119 @@
+import { addToast, dismissToast, ToastType } from '@standardnotes/toast'
+import { ApplicationEvent, InternalEventBus, StorageKey } from '@standardnotes/services'
+import { isDev } from '@/Utils'
+import { FileItem, PrefKey, sleep, SNTag } from '@standardnotes/snjs'
+import { FilesController } from '../FilesController'
+import { preparePhotoOperation, takePhoto, stopCameraStream } from './CameraUtils'
+import { action, makeObservable, observable } from 'mobx'
+import { AbstractViewController } from '@/Controllers/Abstract/AbstractViewController'
+import { WebApplication } from '@/Application/Application'
+import { dateToStringStyle1 } from '@/Utils/DateUtils'
+
+const EVERY_HALF_HOUR = 1000 * 60 * 30
+const EVERY_TEN_SECONDS = 1000 * 10
+const DEBUG_MODE = isDev && false
+
+const DELAY_AFTER_STARTING_CAMERA_TO_ALLOW_MOBILE_AUTOFOCUS = 2000
+
+export class MomentsService extends AbstractViewController {
+  isEnabled = false
+  private intervalReference: ReturnType<typeof setInterval> | undefined
+
+  constructor(application: WebApplication, private filesController: FilesController, eventBus: InternalEventBus) {
+    super(application, eventBus)
+
+    this.disposers.push(
+      application.addEventObserver(async () => {
+        this.isEnabled = (this.application.getValue(StorageKey.MomentsEnabled) as boolean) ?? false
+        if (this.isEnabled) {
+          void this.beginTakingPhotos()
+        }
+      }, ApplicationEvent.Launched),
+    )
+
+    makeObservable(this, {
+      isEnabled: observable,
+
+      enableMoments: action,
+      disableMoments: action,
+    })
+  }
+
+  override deinit() {
+    super.deinit()
+    ;(this.application as unknown) = undefined
+    ;(this.filesController as unknown) = undefined
+  }
+
+  public enableMoments = (): void => {
+    this.application.setValue(StorageKey.MomentsEnabled, true)
+
+    this.isEnabled = true
+
+    void this.beginTakingPhotos()
+  }
+
+  public disableMoments = (): void => {
+    this.application.setValue(StorageKey.MomentsEnabled, false)
+
+    this.isEnabled = false
+
+    clearInterval(this.intervalReference)
+  }
+
+  private beginTakingPhotos() {
+    void this.takePhoto()
+
+    this.intervalReference = setInterval(
+      () => {
+        void this.takePhoto()
+      },
+      DEBUG_MODE ? EVERY_TEN_SECONDS : EVERY_HALF_HOUR,
+    )
+  }
+
+  private getDefaultTag(): SNTag | undefined {
+    const defaultTagId = this.application.getPreference(PrefKey.MomentsDefaultTagUuid)
+
+    if (defaultTagId) {
+      return this.application.items.findItem(defaultTagId)
+    }
+  }
+
+  public async takePhoto(): Promise<FileItem[] | undefined> {
+    const toastId = addToast({
+      type: ToastType.Loading,
+      message: 'Capturing Moment...',
+    })
+
+    const { canvas, video, stream, width, height } = await preparePhotoOperation()
+
+    const filename = `Moment ${dateToStringStyle1(new Date())}.png`
+
+    if (this.application.isMobileDevice) {
+      await sleep(DELAY_AFTER_STARTING_CAMERA_TO_ALLOW_MOBILE_AUTOFOCUS)
+    }
+
+    let file = await takePhoto(filename, canvas, video, width, height)
+    if (!file) {
+      await sleep(1000)
+      file = await takePhoto(filename, canvas, video, width, height)
+      if (!file) {
+        return undefined
+      }
+    }
+
+    dismissToast(toastId)
+
+    const uploadedFile = await this.filesController.uploadNewFile(file)
+
+    const defaultTag = this.getDefaultTag()
+    if (defaultTag && uploadedFile) {
+      void this.application.linkingController.linkItems(uploadedFile[0], defaultTag)
+    }
+
+    stopCameraStream(canvas, video, stream)
+
+    return uploadedFile
+  }
+}