2
0
Thomas Dy 4 жил өмнө
parent
commit
a606611123
11 өөрчлөгдсөн 436 нэмэгдсэн , 50 устгасан
  1. 25 2
      dist/editor.css
  2. 18 13
      dist/editor.html
  3. 1 0
      dist/index.html
  4. 135 9
      src/audio.ts
  5. 40 6
      src/background.ts
  6. 35 7
      src/editor.ts
  7. 4 1
      src/game.ts
  8. 32 12
      src/game/typing.ts
  9. 70 0
      src/global.d.ts
  10. 16 0
      src/util.ts
  11. 60 0
      src/youtube.ts

+ 25 - 2
dist/editor.css

@@ -1,7 +1,16 @@
+body {
+  margin: 0;
+}
+
 #controls {
   position: fixed;
-  width: 90%;
+  width: 100%;
   background-color: white;
+  display: flex;
+}
+
+#controls > div {
+  flex-grow: 1;
 }
 
 .waveform-container {
@@ -51,11 +60,15 @@ li.highlight {
 .text-areas {
   padding-top: 160px;
   display: grid;
-  grid-template-columns: 33% 33% auto;
+  grid-template-columns: auto 1fr 1fr;
   grid-template-rows: 20px auto;
   grid-gap: 2px;
 }
 
+#url {
+  width: 300px;
+}
+
 #intervals-label,
 #kana-label,
 #kanji-label {
@@ -68,6 +81,7 @@ li.highlight {
 
 #intervals,
 #intervals-label {
+  min-width: 250px;
   grid-column: 1 / 2;
 }
 
@@ -97,3 +111,12 @@ li.highlight {
   width: 100%;
   height: 200px;
 }
+
+div#youtube {
+  flex-grow: 0;
+}
+
+iframe#youtube {
+  width: 240px;
+  height: 135px;
+}

+ 18 - 13
dist/editor.html

@@ -6,22 +6,27 @@
   <body>
     <div id="container">
       <div id="controls">
-        <input id="audio" type="file" />
-        <button id="play">Play</button>
-        <button id="pause">Pause</button>
-        <button id="insert-marker">Insert Marker</button>
-        <div class="scrubber">
-          <div class="bar">
-            <div class="bar-overlay"></div>
+        <div>
+          <input id="url" type="text" />
+          <button id="load">Load from url (Youtube/etc)</button>
+          <input id="audio" type="file" />
+          <button id="play">Play</button>
+          <button id="pause">Pause</button>
+          <button id="insert-marker">Insert Marker</button>
+          <div class="scrubber">
+            <div class="bar">
+              <div class="bar-overlay"></div>
+            </div>
+            <div class="markers">
+            </div>
           </div>
-          <div class="markers">
+          <div class="waveform-container">
+            <canvas id="waveform"></canvas>
+            <canvas id="waveform-overlay"></canvas>
           </div>
+          <div id="display"></div>
         </div>
-        <div class="waveform-container">
-          <canvas id="waveform"></canvas>
-          <canvas id="waveform-overlay"></canvas>
-        </div>
-        <div id="display"></div>
+        <div id="youtube"></div>
       </div>
 
       <div class="text-areas">

+ 1 - 0
dist/index.html

@@ -1,6 +1,7 @@
 <html>
   <head>
     <title>Typing Freaks</title>
+    <meta charset="utf-8" />
     <link rel="stylesheet" href="style.css" />
     <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet" />
   </head>

+ 135 - 9
src/audio.ts

@@ -15,14 +15,14 @@ namespace audio {
       return this.context.currentTime;
     }
 
-    async loadTrack(url: string): Promise<Track> {
+    async loadTrack(url: string): Promise<FileTrack> {
       const response = await window.fetch(url);
       const buffer = await response.arrayBuffer();
       const audioBuffer = await this.context.decodeAudioData(buffer);
-      return new Track(this, audioBuffer);
+      return new FileTrack(this, audioBuffer);
     }
 
-    async loadTrackFromFile(file: File): Promise<Track> {
+    async loadTrackFromFile(file: File): Promise<FileTrack> {
       const promise = new Promise<ArrayBuffer>((resolve, _) => {
         const reader = new FileReader();
         reader.onload = () => resolve(reader.result as ArrayBuffer);
@@ -30,26 +30,54 @@ namespace audio {
       });
       const buffer = await promise;
       const audioBuffer = await this.context.decodeAudioData(buffer);
-      return new Track(this, audioBuffer);
+      return new FileTrack(this, audioBuffer);
     }
 
-    async loadTrackWithProgress(url: string, listener: (event: ProgressEvent) => any): Promise<Track> {
+    async loadTrackWithProgress(url: string, listener: (percentage: number) => void): Promise<FileTrack> {
       const promise = new Promise<ArrayBuffer>((resolve, reject) => {
         let xhr = new XMLHttpRequest();
         xhr.open('GET', url);
         xhr.responseType = 'arraybuffer';
-        xhr.onprogress = listener;
+        xhr.onprogress = (event) => {
+          if (event.lengthComputable) {
+            // only up to 80 to factor in decoding time
+            let percentage = event.loaded / event.total * 80;
+            listener(percentage);
+          }
+        };
         xhr.onload = () => resolve(xhr.response);
         xhr.onerror = () => reject();
         xhr.send();
       });
       const buffer = await promise;
       const audioBuffer = await this.context.decodeAudioData(buffer);
-      return new Track(this, audioBuffer);
+      return new FileTrack(this, audioBuffer);
+    }
+
+    async loadTrackFromYoutube(id: string, element: HTMLElement, listener: (percentage: number) => void): Promise<YoutubeTrack> {
+      await youtube.loadYoutubeApi();
+      listener(30);
+      const player = await youtube.createPlayer(element);
+      listener(60);
+      const track = new YoutubeTrack(player, id);
+      await track.preload();
+      listener(90);
+      return track;
     }
   }
 
-  export class Track {
+  export interface Track {
+    play(): void;
+    start(fromTime?: number, duration?: number): void;
+    pause(): void;
+    stop(): void;
+    exit(): void;
+    isPlaying(): boolean;
+    getTime(): number;
+    getDuration(): number;
+  }
+
+  export class FileTrack implements Track {
     manager: AudioManager;
     buffer: AudioBuffer;
     source: AudioBufferSourceNode | null;
@@ -78,7 +106,10 @@ namespace audio {
       this.source.start();
     }
 
-    start(duration?: number): void {
+    start(fromTime?: number, duration?: number): void {
+      if (fromTime !== undefined) {
+        this.resumeTime = fromTime;
+      }
       this.source = this.manager.context.createBufferSource();
       this.source.buffer = this.buffer;
       this.source.connect(this.manager.output);
@@ -114,6 +145,10 @@ namespace audio {
       }
     }
 
+    exit(): void {
+      this.stop();
+    }
+
     isPlaying(): boolean {
       return this.hasStarted && !this.isFinished;
     }
@@ -137,4 +172,95 @@ namespace audio {
       return this.buffer.duration;
     }
   }
+
+  export class YoutubeTrack implements Track {
+    private timeoutHandle?: number;
+    private playDeferred: util.Deferred;
+    private finishDeferred: util.Deferred;
+    readonly fnContext: util.FnContext = new util.FnContext();
+
+    constructor(readonly player: YT.Player, readonly id: string) {
+      this.playDeferred = util.makeDeferred();
+      this.finishDeferred = util.makeDeferred();
+    }
+
+    get playPromise(): Promise<void> {
+      return this.playDeferred.promise;
+    }
+
+    get finishPromise(): Promise<void> {
+      return this.finishDeferred.promise;
+    }
+
+    preload(): Promise<void> {
+      return new Promise((resolve) => {
+        let loaded = false;
+        const onStateChange: YT.PlayerStateChangeListener = ({ data }) => {
+          if (data === YT.PlayerState.PLAYING) {
+            if (!loaded) {
+              loaded = true;
+              this.player.pauseVideo();
+              resolve();
+            }
+          } else if (data === YT.PlayerState.ENDED) {
+            this.finishDeferred.resolve();
+          }
+        };
+        this.player.addEventListener('onStateChange', onStateChange);
+        this.player.loadVideoById(this.id);
+      });
+    }
+
+    play(): void {
+      this.clearTimeout();
+      this.playDeferred.resolve();
+      this.player.playVideo();
+    }
+
+    start(fromTime?: number, duration?: number): void {
+      this.clearTimeout();
+      if (duration) {
+        this.timeoutHandle = setTimeout(() => {
+          this.player.pauseVideo();
+        }, duration * 1000);
+      }
+      if (fromTime !== undefined) {
+        this.player.seekTo(fromTime, true);
+      }
+      this.player.playVideo();
+    }
+
+    pause(): void {
+      this.clearTimeout();
+      this.player.pauseVideo();
+    }
+
+    stop(): void {
+      this.clearTimeout();
+      this.player.stopVideo();
+    }
+
+    exit(): void {
+      // the video will be removed from the background and stop immediately
+      this.fnContext.invalidate();
+    }
+
+    isPlaying(): boolean {
+      return this.player.getPlayerState() === YT.PlayerState.PLAYING;
+    }
+
+    getTime(): number {
+      return this.player.getCurrentTime();
+    }
+
+    getDuration(): number {
+      return this.player.getDuration();
+    }
+
+    private clearTimeout(): void {
+      if (this.timeoutHandle) {
+        clearTimeout(this.timeoutHandle);
+      }
+    }
+  }
 }

+ 40 - 6
src/background.ts

@@ -1,6 +1,7 @@
 namespace background {
   export class BackgroundManager {
     element: HTMLElement;
+    video: HTMLElement;
     last: HTMLElement | null;
     next: HTMLElement;
     fnContext: util.FnContext = new util.FnContext();
@@ -8,15 +9,46 @@ namespace background {
     constructor(element: HTMLElement) {
       this.element = element;
       this.last = null;
+      this.video = document.createElement('div');
+      this.video.classList.add('show');
+      this.element.appendChild(this.video);
       this.next = document.createElement('div');
       this.element.appendChild(this.next);
     }
 
     setBackground(background: string) {
       this.fnContext.invalidate();
-      util.loadBackground(background).then(this.fnContext.wrap(
-        () => this.setBackgroundActual(background)
-      ));
+      util.loadBackground(background).then(this.fnContext.wrap(() => {
+        this.setBackgroundActual(background);
+      }));
+    }
+
+    showVideo() {
+      this.last?.classList.remove('show');
+    }
+
+    hideVideo() {
+      if (this.last != null) {
+        this.last.classList.add('show');
+        this.last.addEventListener('transitionend', () => {
+          this.video.innerHTML = '';
+        });
+      }
+    }
+
+    setVideo(element: HTMLElement) {
+      this.video.innerHTML = '';
+      this.video.appendChild(element);
+    }
+
+    onResize() {
+      const height = this.element.offsetHeight;
+      const width = this.element.offsetWidth;
+      const iframes = this.element.querySelectorAll('iframe');
+      iframes.forEach((iframe) => {
+        iframe.height = ""+height;
+        iframe.width  = ""+width;
+      });
     }
 
     private setBackgroundActual(background: string) {
@@ -29,10 +61,12 @@ namespace background {
       }
       this.next.classList.add('show');
       if (this.last != null) {
+        const toRemove = this.last;
         this.last.classList.remove('show');
-        this.last.addEventListener('transitionend', (event) => {
-          this.element.removeChild(event.target as Node);
-        });
+        this.next.addEventListener('transitionend', () => {
+          this.element.removeChild(toRemove);
+          this.video.innerHTML = '';
+        }, { once: true });
       }
       this.last = this.next;
       this.next = document.createElement('div');

+ 35 - 7
src/editor.ts

@@ -5,6 +5,8 @@
 namespace editor {
   export class Editor {
     audioManager: audio.AudioManager;
+    urlElement: HTMLInputElement;
+    loadElement: HTMLButtonElement;
     audioElement: HTMLInputElement;
     barElement: HTMLElement;
     markerListElement: HTMLElement;
@@ -13,6 +15,7 @@ namespace editor {
     kanjiElement: HTMLTextAreaElement;
     displayElement: HTMLElement;
     jsonElement: HTMLInputElement;
+    waveFormContainer: HTMLDivElement;
     track: audio.Track | null = null;
     markers: Marker[] = [];
     waveForm: WaveForm;
@@ -20,8 +23,14 @@ namespace editor {
 
     constructor() {
       this.audioManager = new audio.AudioManager();
+      this.urlElement = util.getElement(document, '#url');
+      this.loadElement = util.getElement(document, '#load');
+      this.loadElement.addEventListener('click', event => {
+        this.loadAudio();
+      });
       this.audioElement = util.getElement(document, '#audio');
       this.audioElement.addEventListener('change', event => {
+        this.urlElement.value = '';
         this.loadAudio();
       });
       this.barElement = util.getElement(document, '.bar-overlay');
@@ -31,6 +40,7 @@ namespace editor {
       this.kanjiElement = util.getElement(document, '#kanji');
       this.displayElement = util.getElement(document, '#display');
       this.jsonElement = util.getElement(document, '#json');
+      this.waveFormContainer = util.getElement(document, '.waveform-container');
       this.waveForm = new WaveForm(
         util.getElement(document, '#waveform'),
         util.getElement(document, '#waveform-overlay'),
@@ -49,6 +59,20 @@ namespace editor {
     }
 
     loadAudio(): void {
+      const url = this.urlElement.value;
+      if (url != '') {
+        const videoId = youtube.getVideoId(url);
+        if (videoId !== null) {
+          const element = util.getElement(document, '#youtube');
+          this.audioManager.loadTrackFromYoutube(videoId, element, () => {}).then(t => {
+            this.track = t;
+            this.waveForm.clear();
+            this.waveFormContainer.style.display = 'none';
+          });
+          return;
+        }
+      }
+
       let file = this.audioElement.files![0];
       if (file != null) {
         if (this.track != null) {
@@ -58,6 +82,7 @@ namespace editor {
         this.audioManager.loadTrackFromFile(file).then(t => {
           this.track = t;
           this.waveForm.setTrack(t);
+          this.waveFormContainer.style.display = 'block';
         });
       }
     }
@@ -66,7 +91,9 @@ namespace editor {
       if (this.track != null) {
         let percentage = this.track.getTime() / this.track.getDuration() * 100;
         this.barElement.style.width = `${percentage}%`;
-        this.waveForm.update(this.markers);
+        if (this.track instanceof audio.FileTrack) {
+          this.waveForm.update(this.markers);
+        }
         if (this.currentMarker) {
           this.currentMarker.liElement.className = '';
         }
@@ -124,10 +151,7 @@ namespace editor {
 
     play(start?: number, duration?: number): void {
       this.track!.pause();
-      if (start != undefined) {
-        this.track!.resumeTime = start;
-      }
-      this.track!.start(duration);
+      this.track!.start(start, duration);
     }
 
     pause(): void {
@@ -257,7 +281,7 @@ namespace editor {
   class WaveForm {
     ctx: CanvasRenderingContext2D;
     overlayCtx: CanvasRenderingContext2D;
-    track: audio.Track | null = null;
+    track: audio.FileTrack | null = null;
     data: Float32Array | null = null;
     stride: number = 0;
     currentSection: number = -1;
@@ -284,7 +308,11 @@ namespace editor {
       });
     }
 
-    setTrack(track: audio.Track): void {
+    clear(): void {
+      this.track = null;
+    }
+
+    setTrack(track: audio.FileTrack): void {
       this.track = track;
       this.stride = Math.floor(this.track.buffer.sampleRate / this.canvas.width * 5);
       this.currentSection = -1;

+ 4 - 1
src/game.ts

@@ -6,6 +6,7 @@
 
 namespace game {
   export class MainController extends ScreenManager {
+    bgManager: background.BackgroundManager;
     loadingScreen: Screen;
 
     constructor(container: HTMLElement, configUrl: string) {
@@ -14,10 +15,11 @@ namespace game {
 
       let self = this;
       let bgLayer: HTMLElement = util.getElement(container, '#background');
+      this.bgManager = new background.BackgroundManager(bgLayer);
       let gameContext: GameContext = {
         container: container,
         audioManager: new audio.AudioManager(),
-        bgManager: new background.BackgroundManager(bgLayer),
+        bgManager: this.bgManager,
         loadTemplate: (id: string) => util.loadTemplate(container, id),
         assets: null,
         config: null,
@@ -49,6 +51,7 @@ namespace game {
     onResize(): void {
       const fontSize = this.container.offsetHeight / 28.125;
       this.container.style.setProperty('--base-font-size', `${fontSize}px`);
+      this.bgManager.onResize();
     }
   }
 }

+ 32 - 12
src/game/typing.ts

@@ -95,22 +95,42 @@ namespace game {
         this.readyElement.querySelector('.message')!.textContent = 'please wait';
 
         this.fnContext.invalidate();
-        this.context.audioManager.loadTrackWithProgress(
-          this.context.level.audio,
-          this.fnContext.wrap((event: ProgressEvent) => {
-            if (event.lengthComputable) {
-              // only up to 80 to factor in decoding time
-              let percentage = event.loaded / event.total * 80;
-              this.barElement!.style.width = `${percentage}%`;
-            }
-          })
-        ).then(this.fnContext.wrap((track: audio.Track) => {
+
+        const videoId = youtube.getVideoId(this.context.level.audio);
+        const progressListener = this.fnContext.wrap((percentage: number) => {
+          this.barElement!.style.width = `${percentage}%`;
+        });
+        let trackPromise;
+        if (videoId !== null) {
+          const ytElement = document.createElement('div');
+          trackPromise = this.context.audioManager.loadTrackFromYoutube(
+            videoId,
+            ytElement,
+            progressListener,
+          );
+          this.context.bgManager.setVideo(ytElement);
+          if (this.context.level.background == undefined) {
+            trackPromise.then((track) => {
+              track.playPromise.then(track.fnContext.wrap(() => {
+                this.context.bgManager.showVideo();
+              }));
+              track.finishPromise.then(track.fnContext.wrap(() => {
+                this.context.bgManager.hideVideo();
+              }));
+            });
+          }
+        } else {
+          trackPromise = this.context.audioManager.loadTrackWithProgress(
+            this.context.level.audio,
+            progressListener,
+          )
+        }
+        trackPromise.then(this.fnContext.wrap((track: audio.Track) => {
           this.context.track = track;
           this.barElement!.style.width = '100%';
           this.textElement!.textContent = 'music loaded';
           this.setReady();
         }));
-
       } else {
         loader.style.visibility = 'hidden';
         this.setReady();
@@ -331,7 +351,7 @@ namespace game {
 
     exit(): void {
       if (this.context.track !== null) {
-        this.context.track.stop();
+        this.context.track.exit();
       }
     }
 

+ 70 - 0
src/global.d.ts

@@ -0,0 +1,70 @@
+// types for https://developers.google.com/youtube/iframe_api_reference
+// not everything is listed here
+
+declare var onYouTubeIframeAPIReady: () => void;
+
+declare namespace YT {
+  interface PlayerOptions {
+    height: number;
+    width: number;
+    videoId?: string;
+    events?: Partial<PlayerEvents>;
+    playerVars?: Partial<PlayerVars>;
+  }
+
+  type PlayerReadyListener = (event: unknown) => void;
+  type PlayerStateChangeListener = (event: { data: PlayerState }) => void;
+  type PlayerErrorListener = (event: { data: PlayerError }) => void;
+
+  interface PlayerEvents {
+    onReady: PlayerReadyListener;
+    onStateChange: PlayerStateChangeListener;
+    onError: PlayerErrorListener;
+  }
+
+  interface PlayerVars {
+    controls: number;
+    disablekb: number;
+    modestbranding: number;
+    rel: number;
+    iv_load_policy: number;
+    fs: number;
+  }
+
+  enum PlayerState {
+    UNSTARTED = -1,
+    ENDED     = 0,
+    PLAYING   = 1,
+    PAUSED    = 2,
+    BUFFERING = 3,
+    CUED      = 5,
+  }
+
+  enum PlayerError {
+    INVALID_PARAM    = 2,
+    PLAYBACK_ERROR   = 5,
+    NOT_FOUND        = 100,
+    CANNOT_EMBED     = 101,
+    CANNOT_EMBED_ALT = 150,
+  }
+
+  class Player {
+    constructor(element: string | HTMLElement, options: PlayerOptions);
+
+    loadVideoById(id: string, startSeconds?: number): void;
+    cueVideoById(id: string, startSeconds?: number): void;
+    playVideo(): void;
+    pauseVideo(): void;
+    stopVideo(): void;
+    seekTo(seconds: number, allowSeekAhead?: boolean): void;
+
+    getVideoLoadedFraction(): number;
+    getPlayerState(): PlayerState;
+    getCurrentTime(): number;
+    getDuration(): number;
+
+    addEventListener(event: 'onReady', listener: PlayerReadyListener): void;
+    addEventListener(event: 'onStateChange', listener: PlayerStateChangeListener): void;
+    addEventListener(event: 'onError', listener: PlayerErrorListener): void;
+  }
+}

+ 16 - 0
src/util.ts

@@ -104,4 +104,20 @@ namespace util {
       return wrappedFn as any as T;
     }
   }
+
+  export interface Deferred {
+    promise: Promise<void>;
+    resolve: () => void;
+  }
+
+  export function makeDeferred(): Deferred {
+    let resolve: undefined | (() => void);
+    const promise = new Promise<void>((r) => {
+      resolve = r;
+    });
+    return {
+      promise,
+      resolve: resolve!,
+    }
+  }
 }

+ 60 - 0
src/youtube.ts

@@ -0,0 +1,60 @@
+namespace youtube {
+  let apiPromise: Promise<void>;
+
+  export function loadYoutubeApi(): Promise<void> {
+    if (apiPromise) {
+      return apiPromise;
+    }
+    console.time('Loading YouTube API');
+    apiPromise = new Promise((resolve, _) => {
+      const tag = document.createElement('script');
+      tag.src = 'https://www.youtube.com/iframe_api';
+      window.onYouTubeIframeAPIReady = () => {
+        console.timeEnd('Loading YouTube API');
+        resolve();
+      }
+      document.body.appendChild(tag);
+    });
+    return apiPromise;
+  }
+
+  export async function createPlayer(element: HTMLElement): Promise<YT.Player> {
+    await loadYoutubeApi();
+    console.time('Loading YouTube player');
+    return new Promise((resolve, reject) => {
+      const player = new YT.Player(element, {
+        height: element.offsetHeight,
+        width: element.offsetWidth,
+        events: {
+          onReady: () => {
+            console.timeEnd('Loading YouTube player');
+            resolve(player);
+          },
+          onError: ({ data }) => {
+            reject(data);
+          }
+        },
+        playerVars: {
+          controls: 0,
+          disablekb: 1,
+          modestbranding: 1,
+          rel: 0,
+          iv_load_policy: 3,
+          fs: 0
+        }
+      });
+    });
+  }
+
+  export function getVideoId(url: string): string | null {
+    try {
+      const parsed = new URL(url);
+      if (!parsed.hostname.endsWith('youtube.com')) {
+        return null;
+      }
+      return parsed.searchParams.get('v');
+    } catch {
+      return null;
+    }
+  }
+}