Skip to content

Commit

Permalink
fix: #4
Browse files Browse the repository at this point in the history
  • Loading branch information
@p1atdev
p1atdev committed Dec 23, 2022
1 parent e5611df commit f99eb85
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 66 deletions.
4 changes: 2 additions & 2 deletions cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { downloadSearchedMedia, downloadUserMedia } from "./main.ts"

await new Command()
.name("twimedia-wizard")
.version("0.2.0")
.version("0.2.1")
.description("Twitter Media Downloader")
.command("user", "Download media from a user.")
.arguments("<userId:string>")
Expand All @@ -23,7 +23,7 @@ await new Command()
"Download media from Latest tweets. If not specified, it will download media from Top tweets."
)
.action(async ({ output, max, latest }, query) => {
log.info("Search and downloading media from", colors.bold.underline(query))
log.info("Search and download media from", colors.bold.underline(query))
if (latest) {
log.info("Mode:", colors.bold.underline("latest"))
}
Expand Down
103 changes: 43 additions & 60 deletions main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export const getUserMediaUrls = async (restId: string, max = 5000) => {

if (tweets == undefined) {
continue
} else if (Object.keys(tweets).length === 0) {
} else if (tweets.length <= 2) {
break
}

Expand All @@ -44,7 +44,7 @@ export const getUserMediaUrls = async (restId: string, max = 5000) => {
}
case "TimelineTimelineCursor": {
const content = tweet.content as TimelineTimelineCursor
if (content.entryType === "Bottom") {
if (content.cursorType === "Bottom") {
cursor = content.value
}
return []
Expand Down Expand Up @@ -94,7 +94,7 @@ export const searchMediaUrls = async (searchQuery: string, max = 5000, live = fa

if (tweets == undefined) {
continue
} else if (Object.keys(tweets).length === 0) {
} else if (Object.keys(tweets).length <= 2) {
break
}

Expand Down Expand Up @@ -146,6 +146,42 @@ export const searchMediaUrls = async (searchQuery: string, max = 5000, live = fa
return results
}

const donwloadFiles = async (urls: string[], path: string) => {
for (const [index, url] of urls.entries()) {
const filename = url.split("/").pop()

// log.info("Downloading:", filename)

try {
await Deno.open(`${path}/${filename}`)
tty.cursorMove(-1000, 1).text("")
log.warn("Skipping", filename, "because file already exists")
continue
} catch {
tty.eraseLine
.cursorMove(-1000, 0)
.text(`${colors.blue.bold("[INFO]")} Downloading: ${filename} (${index + 1}/${urls.length})`)
}

const res = await fetch(url)
const body = res.body

if (body == null) {
tty.cursorMove(-1000, 1).text("")
log.warn("Skipping", filename, "because body is null")
continue
}

for await (const buffer of body) {
await Deno.writeFile(`${path}/${filename}`, buffer, {
append: true,
})
}
}

tty.eraseLine.cursorMove(-1000, 0).text("")
}

export const downloadUserMedia = async (userId: string, output = "./", max?: number) => {
try {
const restId = await getRestID(userId)
Expand All @@ -155,6 +191,7 @@ export const downloadUserMedia = async (userId: string, output = "./", max?: num

const urls = await getUserMediaUrls(restId, max)

tty.eraseLine.cursorMove(-1000, 0).text("")
log.info("Total count:", urls.length)

// save to output folder
Expand All @@ -168,35 +205,7 @@ export const downloadUserMedia = async (userId: string, output = "./", max?: num
})
}

for (const url of urls) {
const filename = url.split("/").pop()

// log.info("Downloading:", filename)

try {
await Deno.open(`${path}/${filename}`)
tty.cursorMove(-1000, 1).text("")
log.warn("Skipping", filename, "because file already exists")
continue
} catch {
tty.eraseLine.cursorMove(-1000, 0).text(`${colors.blue.bold("[INFO]")} Downloading: ${filename}`)
}

const res = await fetch(url)
const body = res.body

if (body == null) {
tty.cursorMove(-1000, 1).text("")
log.warn("Skipping", filename, "because body is null")
continue
}

for await (const buffer of body) {
await Deno.writeFile(`${path}/${filename}`, buffer, {
append: true,
})
}
}
await donwloadFiles(urls, path)

tty.eraseLine.cursorMove(-1000, 0).text("")

Expand All @@ -214,6 +223,7 @@ export const downloadSearchedMedia = async (searchQuery: string, output = "./",

const urls = await searchMediaUrls(searchQuery, max, live)

tty.eraseLine.cursorMove(-1000, 0).text("")
log.info("Total count:", urls.length)

// save to output folder
Expand All @@ -229,34 +239,7 @@ export const downloadSearchedMedia = async (searchQuery: string, output = "./",
})
}

for (const url of urls) {
const filename = url.split("/").pop()

// log.info("Downloading:", filename)

try {
await Deno.open(`${path}/${filename}`)
tty.cursorMove(-1000, 1).text("")
log.warn("Skipping", filename, "because file already exists")
continue
} catch {
tty.eraseLine.cursorMove(-1000, 0).text(`${colors.blue.bold("[INFO]")} Downloading: ${filename}`)
}

const res = await fetch(url)
const body = res.body

if (body == null) {
log.warn("Skipping", filename, "because body is null")
continue
}

for await (const buffer of body) {
await Deno.writeFile(`${path}/${filename}`, buffer, {
append: true,
})
}
}
await donwloadFiles(urls, path)

tty.eraseLine.cursorMove(-1000, 0).text("")

Expand Down
36 changes: 36 additions & 0 deletions tests/main.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,35 @@ Deno.test("get a lot of media urls", async () => {

assertExists(restId)

const count = 500
const urls = await getUserMediaUrls(restId, count)

assertEquals(urls.length, count)
})

Deno.test("get exceeded amount of media urls", async () => {
const restId = await getRestID(userId)

assertExists(restId)

const count = 1000
const urls = await getUserMediaUrls(restId, count)

assertNotEquals(urls.length, count)
})

Deno.test("get a lot of media urls, and check conflicting", async () => {
const restId = await getRestID(userId)

assertExists(restId)

const count = 300
const urls = await getUserMediaUrls(restId, count)

// is thare same url?
const uniqueUrls = [...new Set(urls)]

assertEquals(urls.length, uniqueUrls.length)
assertEquals(urls.length, count)
})

Expand All @@ -42,6 +68,16 @@ Deno.test("search a lot of media urls", async () => {
assertEquals(urls.length, count)
})

Deno.test("search a lot of media urls, and check conflicting", async () => {
const count = 150
const urls = await searchMediaUrls(searchQuery, count)

// is thare same url?
const uniqueUrls = [...new Set(urls)]

assertEquals(urls.length, uniqueUrls.length)
})

Deno.test("search media urls in live", async () => {
const count = 10
const urls = await searchMediaUrls(searchQuery, count)
Expand Down
5 changes: 1 addition & 4 deletions utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export const getUserTweets = async (userId: string, cursor?: string) => {
withSuperFollowsTweetFields: true,
withVoice: true,
withV2Timeline: true,
cursor: cursor,
},
features: {
responsive_web_twitter_blue_verified_badge_is_enabled: true,
Expand All @@ -90,10 +91,6 @@ export const getUserTweets = async (userId: string, cursor?: string) => {
},
})

if (cursor) {
query.data.cursor = cursor
}

const res = await client.request({
method: "GET",
urlType: "gql",
Expand Down

0 comments on commit f99eb85

Please sign in to comment.