diff --git a/cli.ts b/cli.ts index 893a58b..355e930 100644 --- a/cli.ts +++ b/cli.ts @@ -4,7 +4,7 @@ import { downloadSearchedMedia, downloadUserMedia } from "./main.ts" await new Command() .name("twimedia-wizard") - .version("0.2.0") + .version("0.2.1") .description("Twitter Media Downloader") .command("user", "Download media from a user.") .arguments("") @@ -23,7 +23,7 @@ await new Command() "Download media from Latest tweets. If not specified, it will download media from Top tweets." ) .action(async ({ output, max, latest }, query) => { - log.info("Search and downloading media from", colors.bold.underline(query)) + log.info("Search and download media from", colors.bold.underline(query)) if (latest) { log.info("Mode:", colors.bold.underline("latest")) } diff --git a/main.ts b/main.ts index 007a3b7..c75b8a9 100644 --- a/main.ts +++ b/main.ts @@ -24,7 +24,7 @@ export const getUserMediaUrls = async (restId: string, max = 5000) => { if (tweets == undefined) { continue - } else if (Object.keys(tweets).length === 0) { + } else if (tweets.length <= 2) { break } @@ -44,7 +44,7 @@ export const getUserMediaUrls = async (restId: string, max = 5000) => { } case "TimelineTimelineCursor": { const content = tweet.content as TimelineTimelineCursor - if (content.entryType === "Bottom") { + if (content.cursorType === "Bottom") { cursor = content.value } return [] @@ -94,7 +94,7 @@ export const searchMediaUrls = async (searchQuery: string, max = 5000, live = fa if (tweets == undefined) { continue - } else if (Object.keys(tweets).length === 0) { + } else if (Object.keys(tweets).length <= 2) { break } @@ -146,6 +146,42 @@ export const searchMediaUrls = async (searchQuery: string, max = 5000, live = fa return results } +const donwloadFiles = async (urls: string[], path: string) => { + for (const [index, url] of urls.entries()) { + const filename = url.split("/").pop() + + // log.info("Downloading:", filename) + + try { + await Deno.open(`${path}/${filename}`) + tty.cursorMove(-1000, 1).text("") + log.warn("Skipping", filename, "because file already exists") + continue + } catch { + tty.eraseLine + .cursorMove(-1000, 0) + .text(`${colors.blue.bold("[INFO]")} Downloading: ${filename} (${index + 1}/${urls.length})`) + } + + const res = await fetch(url) + const body = res.body + + if (body == null) { + tty.cursorMove(-1000, 1).text("") + log.warn("Skipping", filename, "because body is null") + continue + } + + for await (const buffer of body) { + await Deno.writeFile(`${path}/${filename}`, buffer, { + append: true, + }) + } + } + + tty.eraseLine.cursorMove(-1000, 0).text("") +} + export const downloadUserMedia = async (userId: string, output = "./", max?: number) => { try { const restId = await getRestID(userId) @@ -155,6 +191,7 @@ export const downloadUserMedia = async (userId: string, output = "./", max?: num const urls = await getUserMediaUrls(restId, max) + tty.eraseLine.cursorMove(-1000, 0).text("") log.info("Total count:", urls.length) // save to output folder @@ -168,35 +205,7 @@ export const downloadUserMedia = async (userId: string, output = "./", max?: num }) } - for (const url of urls) { - const filename = url.split("/").pop() - - // log.info("Downloading:", filename) - - try { - await Deno.open(`${path}/${filename}`) - tty.cursorMove(-1000, 1).text("") - log.warn("Skipping", filename, "because file already exists") - continue - } catch { - tty.eraseLine.cursorMove(-1000, 0).text(`${colors.blue.bold("[INFO]")} Downloading: ${filename}`) - } - - const res = await fetch(url) - const body = res.body - - if (body == null) { - tty.cursorMove(-1000, 1).text("") - log.warn("Skipping", filename, "because body is null") - continue - } - - for await (const buffer of body) { - await Deno.writeFile(`${path}/${filename}`, buffer, { - append: true, - }) - } - } + await donwloadFiles(urls, path) tty.eraseLine.cursorMove(-1000, 0).text("") @@ -214,6 +223,7 @@ export const downloadSearchedMedia = async (searchQuery: string, output = "./", const urls = await searchMediaUrls(searchQuery, max, live) + tty.eraseLine.cursorMove(-1000, 0).text("") log.info("Total count:", urls.length) // save to output folder @@ -229,34 +239,7 @@ export const downloadSearchedMedia = async (searchQuery: string, output = "./", }) } - for (const url of urls) { - const filename = url.split("/").pop() - - // log.info("Downloading:", filename) - - try { - await Deno.open(`${path}/${filename}`) - tty.cursorMove(-1000, 1).text("") - log.warn("Skipping", filename, "because file already exists") - continue - } catch { - tty.eraseLine.cursorMove(-1000, 0).text(`${colors.blue.bold("[INFO]")} Downloading: ${filename}`) - } - - const res = await fetch(url) - const body = res.body - - if (body == null) { - log.warn("Skipping", filename, "because body is null") - continue - } - - for await (const buffer of body) { - await Deno.writeFile(`${path}/${filename}`, buffer, { - append: true, - }) - } - } + await donwloadFiles(urls, path) tty.eraseLine.cursorMove(-1000, 0).text("") diff --git a/tests/main.test.ts b/tests/main.test.ts index e6faf12..b9a3e03 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -20,9 +20,35 @@ Deno.test("get a lot of media urls", async () => { assertExists(restId) + const count = 500 + const urls = await getUserMediaUrls(restId, count) + + assertEquals(urls.length, count) +}) + +Deno.test("get exceeded amount of media urls", async () => { + const restId = await getRestID(userId) + + assertExists(restId) + const count = 1000 const urls = await getUserMediaUrls(restId, count) + assertNotEquals(urls.length, count) +}) + +Deno.test("get a lot of media urls, and check conflicting", async () => { + const restId = await getRestID(userId) + + assertExists(restId) + + const count = 300 + const urls = await getUserMediaUrls(restId, count) + + // is thare same url? + const uniqueUrls = [...new Set(urls)] + + assertEquals(urls.length, uniqueUrls.length) assertEquals(urls.length, count) }) @@ -42,6 +68,16 @@ Deno.test("search a lot of media urls", async () => { assertEquals(urls.length, count) }) +Deno.test("search a lot of media urls, and check conflicting", async () => { + const count = 150 + const urls = await searchMediaUrls(searchQuery, count) + + // is thare same url? + const uniqueUrls = [...new Set(urls)] + + assertEquals(urls.length, uniqueUrls.length) +}) + Deno.test("search media urls in live", async () => { const count = 10 const urls = await searchMediaUrls(searchQuery, count) diff --git a/utils.ts b/utils.ts index 06b5b13..7705d6e 100644 --- a/utils.ts +++ b/utils.ts @@ -70,6 +70,7 @@ export const getUserTweets = async (userId: string, cursor?: string) => { withSuperFollowsTweetFields: true, withVoice: true, withV2Timeline: true, + cursor: cursor, }, features: { responsive_web_twitter_blue_verified_badge_is_enabled: true, @@ -90,10 +91,6 @@ export const getUserTweets = async (userId: string, cursor?: string) => { }, }) - if (cursor) { - query.data.cursor = cursor - } - const res = await client.request({ method: "GET", urlType: "gql",