Initial commit

.gitignore

@@ -0,0 +1,2 @@
+config.json
+*.db

Makefile

@@ -0,0 +1,10 @@
+PHONY: dev
+
+dev:
+	deno run \
+		--config tsconfig.json \
+		--watch \
+		--allow-net \
+		--allow-read=config.json,store.db,store.db-journal \
+		--allow-write=store.db,store.db-journal \
+		index.ts

config.json.stub

@@ -0,0 +1,4 @@
+{
+  "bearer": "",
+  "store": "store.db"
+}

format.ts

@@ -0,0 +1,262 @@
+import { Tweet } from './twitter.ts';
+
+class SafeString {
+  constructor(readonly raw: string) {}
+
+  get length(): number {
+    return this.raw.length;
+  }
+
+  toString(): string {
+    return this.raw;
+  }
+
+  toJSON(): string {
+    return this.raw;
+  }
+}
+
+type StringLike = string | SafeString;
+
+function escapeHTML(unsafe: string): StringLike {
+  return new SafeString(
+    unsafe
+      .replace(/&/g, "&")
+      .replace(/</g, "&lt;")
+      .replace(/>/g, "&gt;")
+      .replace(/"/g, "&quot;")
+      .replace(/'/g, "&#039;")
+  );
+}
+
+function joinChildren(children: StringLike[]): SafeString {
+  return new SafeString(children
+    .map(child => typeof child === 'string' ? escapeHTML(child) : child)
+    .join('')
+  );
+}
+
+function tag(tag: string, attributes: Record<string, string> = {}, children: StringLike | StringLike[] = []): SafeString {
+  const attrs = Object.entries(attributes).map(([ key, value ]) => {
+    return ` ${key}="${escapeHTML(value)}"`;
+  }).join('');
+  if (children.length === 0) {
+    return new SafeString(`<${tag}${attrs} />`);
+  } else {
+    const childrenArray = Array.isArray(children) ? children : [children];
+    const normalizedChildren = joinChildren(childrenArray);
+    return new SafeString(`<${tag}${attrs}>${normalizedChildren}</${tag}>`);
+  }
+}
+
+function buildTwitterUrl(url: string): string {
+  return new URL(url, 'https://twitter.com').toString();
+}
+
+function buildProxyUrl(url: string): string {
+  const search = new URLSearchParams({ target: url }).toString();
+  return `/__proxy?${search}`;
+}
+
+function formatPlainText(text: string): SafeString {
+  return new SafeString(escapeHTML(text).toString().replace(/\n/g, "<br />"));
+}
+
+class TextFormatter {
+  private splices: { text: StringLike, indices: [number, number] }[];
+  private media: { type: 'video' | 'img', url: string, link?: string }[];
+  private characters: string[];
+
+  constructor(readonly tweet: Tweet, readonly useProxy: boolean) {
+    this.characters = [...tweet.full_text];
+    this.splices = [];
+    this.media = [];
+    for (const { indices, text } of tweet.entities.hashtags) {
+      const url = buildTwitterUrl(`/hashtag/${text}`);
+      this.splices.push({
+        indices,
+        text: tag('a', { href: url }, `#${text}`),
+      });
+    }
+    for (const link of tweet.entities.urls) {
+      const url = new URL(link.expanded_url).toString();
+      this.splices.push({
+        indices: link.indices,
+        text: tag('a', { href: url }, link.display_url),
+      });
+    }
+    for (const { indices, name, screen_name } of tweet.entities.user_mentions) {
+      const url = buildTwitterUrl(`/${screen_name}`);
+      this.splices.push({
+        indices: indices,
+        text: tag('a', { href: url, title: name }, `@${screen_name}`),
+      });
+    }
+    const media = tweet.extended_entities?.media ?? [];
+    for (const item of media) {
+      if (item.type === 'photo') {
+        const url = new URL(item.media_url_https).toString();
+        this.media.push({ type: 'img', url });
+      } else if (item.video_info !== undefined) {
+        let max = -1;
+        let maxUrl: string | undefined = undefined;
+        for (const variant of item.video_info.variants) {
+          if (variant.bitrate === undefined) {
+            continue;
+          }
+          if (variant.bitrate > max) {
+            max = variant.bitrate;
+            maxUrl = variant.url;
+          }
+        }
+        if (maxUrl !== undefined) {
+          const url = new URL(maxUrl).toString();
+          this.media.push({ type: 'video', url });
+        } else {
+          const url = new URL(item.media_url_https).toString();
+          this.media.push({ type: 'img', url, link: item.expanded_url });
+        }
+      }
+    }
+  }
+
+  getRange(start: number, end?: number): string {
+    const max = this.tweet.display_text_range[1];
+    return this.characters.slice(start, end ?? max).join('');
+  }
+
+  headerHTML(): SafeString {
+    const date = new Date(this.tweet.created_at);
+    const dateOptions = {
+      weekday: 'short',
+      year: 'numeric',
+      month: '2-digit',
+      day: '2-digit',
+      hour: '2-digit',
+      minute: '2-digit',
+      timeZoneName: 'short',
+    } as const;
+    const imageUrl = new URL(this.tweet.user.profile_image_url_https).toString();
+    const imageSrc = this.useProxy ? buildProxyUrl(imageUrl) : imageUrl;
+    const profileUrl = buildTwitterUrl(`/${this.tweet.user.screen_name}`);
+    const tweetUrl = buildTwitterUrl(`/${this.tweet.user.screen_name}/status/${this.tweet.id_str}`);
+    const html = [
+      tag('img', { loading: 'lazy', src: imageSrc, height: '24px', width: '24px' }),
+      ' ',
+      tag('strong', {}, this.tweet.user.name),
+      ' ',
+      tag('a', { href: profileUrl }, `@${this.tweet.user.screen_name}`),
+      tag('br'),
+      'Posted ',
+      tag('a', { href: tweetUrl }, date.toLocaleString(this.tweet.lang, dateOptions)),
+    ];
+    return joinChildren(html);
+  }
+
+  bodyHTML(): SafeString {
+    const max = this.tweet.display_text_range[1];
+    const splices = this.splices
+      .filter(({ indices }) => indices[0] < max && indices[1] <= max)
+      .sort((a, b) => a.indices[0] - b.indices[0]);
+
+    let index = 0;
+    const html: StringLike[] = [];
+    for (const { text, indices } of splices) {
+      const start = index;
+      const end = indices[0];
+
+      html.push(formatPlainText(this.getRange(start, end)));
+      html.push(text);
+
+      index = indices[1];
+    }
+    html.push(formatPlainText(this.getRange(index)));
+    for (const { type, url, link } of this.media) {
+      html.push(tag('br'));
+      html.push(tag('br'));
+      const src = this.useProxy ? buildProxyUrl(url) : url;
+      if (type === 'img') {
+        html.push(tag('a', { href: link ?? url }, [
+          tag('img', { loading: 'lazy', src }),
+        ]));
+      } else if (type === 'video') {
+        html.push(tag('video', { controls: '', src }));
+      }
+    }
+    return joinChildren(html);
+  }
+
+  toHTML(): SafeString {
+    return joinChildren([ this.headerHTML(), tag('br'), this.bodyHTML() ]);
+  }
+}
+
+const STYLES = `
+body > div {
+  margin: 10px;
+  padding: 10px;
+  border: solid 1px gray;
+  border-radius: 10px;
+}
+blockquote {
+  padding: 10px;
+  border: solid 1px lightgray;
+  border-radius: 10px;
+}
+div {
+  max-width: 600px;
+}
+img {
+  max-width: 100%;
+}
+`;
+
+export function timelineAsHTML(tweets: Tweet[]): string {
+  const body = tweets.map(tweet => {
+    const displayTweet = tweet.retweeted_status ?? tweet;
+    const children: StringLike[] = [];
+    children.push(new TextFormatter(displayTweet, true).toHTML());
+
+    const quoteTweet = displayTweet.quoted_status;
+    if (quoteTweet !== undefined) {
+      children.push(tag('blockquote', {}, new TextFormatter(quoteTweet, true).toHTML()));
+    }
+    return tag('div', {}, children);
+  }).join('\n');
+  return `
+    <html>
+      <head>
+        <style>${STYLES}</style>
+      </head>
+      <body>${body}</body>
+    </html>
+  `;
+}
+
+export function timelineAsJSON(username: string, tweets: Tweet[]): string {
+  const items = tweets.map(tweet => {
+    const displayTweet = tweet.retweeted_status ?? tweet;
+    const children: StringLike[] = [];
+    children.push(new TextFormatter(displayTweet, false).bodyHTML());
+
+    const quoteTweet = displayTweet.quoted_status;
+    if (quoteTweet !== undefined) {
+      children.push(tag('blockquote', {}, [
+        new TextFormatter(quoteTweet, false).toHTML(),
+      ]));
+    }
+    const html = joinChildren(children);
+    return {
+      id: tweet.id_str,
+      url: buildTwitterUrl(`/${tweet.user.screen_name}/status/${tweet.id_str}`),
+      content_html: html,
+      date_published: new Date(tweet.created_at).toISOString(),
+    };
+  });
+  return JSON.stringify({
+    version: '1.1',
+    title: `Twitter @${username}`,
+    home_page_url: buildTwitterUrl(`/${username}`),
+    items,
+  });
+}

index.ts

@@ -0,0 +1,143 @@
+import { Tweet, TwitterClient } from './twitter.ts';
+import { SqliteStore, Store } from './store.ts';
+import { StaticTokenProvider } from './token.ts';
+import { timelineAsHTML, timelineAsJSON } from './format.ts';
+import { z } from "https://deno.land/x/zod@v3.8.0/mod.ts";
+
+const MAX_INT = "9223372036854775807";
+
+class Server {
+  constructor(
+    readonly store: Store,
+    readonly client: TwitterClient,
+  ) {}
+
+  async getLatestTweets(username: string, interval: number = 600): Promise<Tweet[]> {
+    const storedAuthorId = this.store.getAuthorId(username);
+    const latestTweet = storedAuthorId === null ? null : this.store.getLatestTweet(storedAuthorId);
+
+    let timeline = await this.client.getTimeline(username, latestTweet?.id_str);
+    console.log(`Fetched ${timeline.length} new tweets`);
+    for (const tweet of timeline) {
+      this.store.saveTweet(tweet);
+    }
+
+    const newerThan = new Date();
+    newerThan.setSeconds(newerThan.getSeconds() - interval);
+
+    if (timeline.length === 0) {
+      if (storedAuthorId === null) {
+        return [];
+      } else {
+        timeline = this.store.getLatestTweets(storedAuthorId, MAX_INT, newerThan);
+      }
+    } else {
+      const lastTweet = timeline[timeline.length - 1];
+      const olderTweets = this.store.getLatestTweets(lastTweet.user.id_str, lastTweet.id_str, newerThan);
+      timeline = timeline.concat(olderTweets);
+    }
+    return timeline;
+  }
+
+  async start() {
+    const server = Deno.listen({ port: 8000 });
+    console.log("Server running at http://localhost:8000");
+
+    for await (const conn of server) {
+      this.serveHttp(conn);
+    }
+  }
+
+  async serveHttp(conn: Deno.Conn) {
+    const httpConn = Deno.serveHttp(conn);
+    for await (const requestEvent of httpConn) {
+      this.handle(requestEvent)
+        .then(response => requestEvent.respondWith(response))
+        .catch(err => console.error(err));
+    }
+  }
+
+  async handle(requestEvent: Deno.RequestEvent): Promise<Response> {
+    const url = new URL(requestEvent.request.url);
+    const path = url.pathname;
+    if (path === '/__proxy') {
+      const target = url.searchParams.get('target');
+      return this.handleProxy(requestEvent.request.headers, target);
+    } else if (path.startsWith('/timeline/')) {
+      const username = path.substring(10);
+      return this.handleTimeline(username);
+    } else if (path.startsWith('/json/')) {
+      const username = path.substring(6);
+      return this.handleJson(username);
+    } else {
+      return new Response('Not found', {
+        status: 404,
+      });
+    }
+  }
+
+  async handleTimeline(username: string): Promise<Response> {
+    const tweets = await this.getLatestTweets(username, 24 * 60 * 60);
+    console.time('renderHTML');
+    const html = timelineAsHTML(tweets);
+    console.timeEnd('renderHTML');
+    return new Response(html, {
+      status: 200,
+      headers: {
+        'Content-Type': 'text/html; charset=utf-8'
+      }
+    });
+  }
+
+  async handleJson(username: string): Promise<Response> {
+    const tweets = await this.getLatestTweets(username, 24 * 60 * 60);
+    console.time('renderJSON');
+    const html = timelineAsJSON(username, tweets);
+    console.timeEnd('renderJSON');
+    return new Response(html, {
+      status: 200,
+      headers: {
+        'Content-Type': 'application/feed+json; charset=utf-8'
+      }
+    });
+  }
+
+  async handleProxy(headers: Headers, target: string | null): Promise<Response> {
+    if (target !== null) {
+      console.log(`Proxying ${target}`);
+      try {
+        const proxyRequest = await fetch(target, {
+          headers: headers,
+        });
+        return new Response(proxyRequest.body, {
+          status: proxyRequest.status,
+          headers: proxyRequest.headers,
+        });
+      } catch (e) {
+        return new Response('Bad request', {
+          status: 400,
+        });
+      }
+    } else {
+      return new Response('Not found', {
+        status: 404,
+      });
+    }
+  }
+}
+
+const ConfigSchema = z.strictObject({
+  bearer: z.string(),
+  store: z.string(),
+});
+
+const configFile = Deno.args.length === 0 ? 'config.json' : Deno.args[0];
+const json = JSON.parse(Deno.readTextFileSync(configFile));
+const config = ConfigSchema.parse(json);
+
+const fetcher = new Server(
+  new SqliteStore(config.store),
+  new TwitterClient(new StaticTokenProvider(config.bearer)),
+);
+
+fetcher.start();

store.ts

@@ -0,0 +1,114 @@
+import { DB } from "https://deno.land/x/sqlite@v3.1.1/mod.ts";
+import { User, Tweet, TweetSchema } from "./twitter.ts";
+
+export abstract class Store {
+  saveTweet(tweet: Tweet) {
+    this.saveSingleTweet(tweet);
+    this.saveUser(tweet.user);
+    if (tweet.quoted_status !== undefined) {
+      this.saveTweet(tweet.quoted_status);
+    }
+    if (tweet.retweeted_status !== undefined) {
+      this.saveTweet(tweet.retweeted_status);
+    }
+  }
+
+  abstract saveSingleTweet(tweet: Tweet): void;
+  abstract saveUser(user: User): void;
+  abstract getLatestTweet(authorId: string): Tweet | null;
+  abstract getLatestTweets(authorId: string, olderThanId: string, newerThan: Date): Tweet[];
+  abstract getAuthorId(username: string): string | null;
+}
+
+function dateToEpoch(dateStr: string | Date) {
+  const date = typeof dateStr === 'string' ? new Date(dateStr) : dateStr;
+  return Math.floor(date.getTime() / 1000);
+}
+
+export class SqliteStore extends Store {
+  private db: DB;
+
+  constructor(path: string) {
+    super();
+    this.db = new DB(path);
+    this.db.query(`
+      CREATE TABLE IF NOT EXISTS tweets (
+        id INTEGER PRIMARY KEY,
+        author_id INTEGER NOT NULL,
+        created_at INTEGER NOT NULL,
+        tweet TEXT NOT NULL,
+        fetched_at INTEGER NOT NULL
+      );
+    `);
+    this.db.query(`
+      CREATE TABLE IF NOT EXISTS users (
+        id INTEGER PRIMARY KEY,
+        username TEXT NOT NULL UNIQUE
+      );
+    `);
+  }
+
+  saveSingleTweet(tweet: Tweet) {
+    this.db.query("INSERT OR IGNORE INTO tweets VALUES (?, ?, ?, ?, ?)", [
+      BigInt(tweet.id_str),
+      BigInt(tweet.user.id_str),
+      dateToEpoch(tweet.created_at),
+      JSON.stringify(tweet),
+      dateToEpoch(new Date()),
+    ]);
+  }
+
+  saveUser(user: User) {
+    this.db.query("INSERT OR REPLACE INTO users VALUES (?, ?)", [
+      BigInt(user.id_str),
+      user.screen_name,
+    ]);
+  }
+
+  getLatestTweet(authorId: string): Tweet | null {
+    const result = this.db.query<[string]>(`
+      SELECT tweet FROM tweets
+      WHERE author_id = ?
+      ORDER BY id DESC
+      LIMIT 1
+    `, [ BigInt(authorId) ]);
+
+    if (result.length === 0) {
+      return null;
+    } else {
+      const tweet = JSON.parse(result[0][0]);
+      return TweetSchema.parse(tweet);
+    }
+  }
+
+  getLatestTweets(authorId: string, olderThanId: string, newerThan: Date): Tweet[] {
+    const result = this.db.query<[string]>(`
+      SELECT tweet FROM tweets
+      WHERE author_id = ?
+        AND id < ?
+        AND created_at > ?
+      ORDER BY id DESC
+    `, [ BigInt(authorId), BigInt(olderThanId), dateToEpoch(newerThan) ]);
+
+    return result.map(([ tweetStr ]) => {
+      const tweet = JSON.parse(tweetStr);
+      return TweetSchema.parse(tweet);
+    });
+  }
+
+  getAuthorId(username: string): string | null {
+    const result = this.db.query<[BigInt]>(`
+      SELECT id FROM users
+      WHERE username = ?
+      ORDER BY id DESC
+      LIMIT 1
+    `, [ username ]);
+
+
+    if (result.length === 0) {
+      return null;
+    } else {
+      return result[0][0].toString();
+    }
+  }
+}

token.ts

@@ -0,0 +1,11 @@
+export interface TokenProvider {
+  getToken(): Promise<string>;
+}
+
+export class StaticTokenProvider implements TokenProvider {
+  constructor(private readonly cachedToken: string) {}
+
+  async getToken(): Promise<string> {
+    return this.cachedToken;
+  }
+}

tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "compilerOptions": {
+    "strictPropertyInitialization": false
+  }
+}

twitter.ts

@@ -0,0 +1,115 @@
+import { z } from "https://deno.land/x/zod@v3.8.0/mod.ts";
+import { TokenProvider } from "./token.ts";
+
+const TWITTER_API = 'https://api.twitter.com';
+
+function object<T extends z.ZodRawShape>(shape: T): z.ZodObject<T, "passthrough"> {
+  return z.object(shape).passthrough();
+}
+
+const IndicesSchema = z.tuple([ z.number(), z.number() ]);
+
+const BaseUserSchema = object({
+  id_str: z.string(),
+  name: z.string(),
+  screen_name: z.string(),
+});
+
+const UserSchema = BaseUserSchema.extend({
+  profile_image_url_https: z.string(),
+});
+
+export type User = z.infer<typeof UserSchema>;
+
+const HashtagSchema = object({
+  text: z.string(),
+  indices: IndicesSchema,
+});
+
+const UrlSchema = object({
+  display_url: z.string(),
+  expanded_url: z.string(),
+  url: z.string(),
+  indices: IndicesSchema,
+});
+
+const BaseTweetSchema = object({
+  id_str: z.string(),
+  created_at: z.string(),
+  full_text: z.string(),
+  user: UserSchema,
+  display_text_range: IndicesSchema,
+  lang: z.optional(z.string()),
+  entities: object({
+    hashtags: z.array(HashtagSchema),
+    urls: z.array(UrlSchema),
+    user_mentions: z.array(BaseUserSchema.extend({
+      indices: IndicesSchema,
+    })),
+  }),
+  is_quote_status: z.boolean(),
+  extended_entities: z.optional(object({
+    media: z.array(object({
+      display_url: z.string(),
+      media_url_https: z.string(),
+      expanded_url: z.string(),
+      indices: IndicesSchema,
+      type: z.enum(['photo', 'video', 'animated_gif']),
+      video_info: z.optional(object({
+        variants: z.array(object({
+          bitrate: z.optional(z.number()),
+          content_type: z.string(),
+          url: z.string(),
+        })),
+      })),
+    })),
+  })),
+});
+
+interface TweetRecurse {
+  quoted_status?: Tweet,
+  retweeted_status?: Tweet,
+}
+
+export type Tweet = z.infer<typeof BaseTweetSchema> & TweetRecurse;
+
+export const TweetSchema: z.ZodSchema<Tweet> = z.lazy(() => BaseTweetSchema.extend({
+  quoted_status: z.optional(TweetSchema),
+  retweeted_status: z.optional(TweetSchema),
+}));
+
+export const TimelineSchema = z.array(TweetSchema);
+
+export class TwitterClient {
+  constructor(private readonly tokenProvider: TokenProvider) {}
+
+  async getTimeline(username: string, newerThanId?: string): Promise<Tweet[]> {
+    const options: Record<string, string> = {
+      screen_name: username,
+      trim_user: 'false',
+      exclude_replies: 'true',
+      include_rts: 'true',
+      tweet_mode: 'extended',
+    };
+    if (newerThanId !== undefined) {
+      options.since_id = newerThanId;
+      options.count = '100';
+    }
+    const result = await this.twitterGet(`/1.1/statuses/user_timeline.json`, options);
+    return TimelineSchema.parse(result);
+  }
+
+  private async twitterGet(path: string, params: Record<string, string> = {}): Promise<unknown> {
+    const url = new URL(`${TWITTER_API}${path}`);
+    url.search = new URLSearchParams(params).toString();
+    const result = await fetch(url, {
+      headers: {
+        Authorization: `Bearer ${await this.tokenProvider.getToken()}`
+      }
+    });
+    if (result.status < 200 || result.status >= 300) {
+      throw new Error(await result.text());
+    }
+    return result.json();
+  }
+}