From 4f8b788a05305ddcdfc884fdf11e5dcb46c3daa5 Mon Sep 17 00:00:00 2001 From: tangyoha <39958403+tangyoha@users.noreply.github.com> Date: Wed, 7 Dec 2022 22:08:06 +0800 Subject: [PATCH] refactor:provide save path option (#13) * refactor:provide file save path option Co-authored-by: tangyoha --- .github/FUNDING.yml | 2 - .../documentation-improvement.md | 2 +- CONTRIBUTING.md | 10 +- README.md | 107 +++++++---- README_CN.md | 179 ++++++++++++++++++ config.yaml | 36 ++-- media_downloader.py | 76 ++++---- module/app.py | 45 ++++- requirements.txt | 2 +- setup.py | 12 +- tests/test_app/test_app.py | 16 +- tests/test_media_downloader.py | 36 ++-- tests/utils/test_updates.py | 6 +- utils/__init__.py | 4 +- 14 files changed, 396 insertions(+), 137 deletions(-) delete mode 100644 .github/FUNDING.yml create mode 100644 README_CN.md diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 891c6f87..00000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,2 +0,0 @@ -github: Dineshkarthik -custom: ["https://www.buymeacoffee.com/dkraveendran"] diff --git a/.github/ISSUE_TEMPLATE/documentation-improvement.md b/.github/ISSUE_TEMPLATE/documentation-improvement.md index 64fd284b..f035739b 100644 --- a/.github/ISSUE_TEMPLATE/documentation-improvement.md +++ b/.github/ISSUE_TEMPLATE/documentation-improvement.md @@ -9,7 +9,7 @@ assignees: '' #### Location of the documentation -[this should provide the location of the documentation, e.g. "CONTRIBUTION.md" or the URL of the documentation, e.g. "https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CONTRIBUTING.md"] +[this should provide the location of the documentation, e.g. "CONTRIBUTION.md" or the URL of the documentation, e.g. "https://github.com/tangyoha/telegram_media_downloader/blob/master/CONTRIBUTING.md"] #### Documentation problem diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b03fa21..4cc2c7e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,9 +5,9 @@ Please take a moment to review this document in order to make the contribution p ### Where do I go from here? -If you've noticed a bug or have a feature request, [make one](https://github.com/Dineshkarthik/telegram_media_downloader/issues)! It's generally best if you get confirmation of your bug or approval for your feature request this way before starting to code. +If you've noticed a bug or have a feature request, [make one](https://github.com/tangyoha/telegram_media_downloader/issues)! It's generally best if you get confirmation of your bug or approval for your feature request this way before starting to code. -If you have a general question about telegram-media-downloader, you can ask it on [Discussion](https://github.com/Dineshkarthik/telegram_media_downloader/discussions) under `Q&A` category and any ideas/suggestions goes under `Ideas` category, the issue tracker is only for bugs and feature requests. +If you have a general question about telegram-media-downloader, you can ask it on [Discussion](https://github.com/tangyoha/telegram_media_downloader/discussions) under `Q&A` category and any ideas/suggestions goes under `Ideas` category, the issue tracker is only for bugs and feature requests. ### Fork & create a branch @@ -66,13 +66,13 @@ Please follow these coding standards when writing code for inclusion in telegram Telegram-media-downloader follows the [PEP8](https://www.python.org/dev/peps/pep-0008/) standard and uses [Black](https://black.readthedocs.io/en/stable/) and [Pylint](https://pylint.pycqa.org/en/latest/) to ensure a consistent code format throughout the project. -[Continuous Integration](https://github.com/Dineshkarthik/telegram_media_downloader/actions) using GitHub Actions will run those tools and report any stylistic errors in your code. Therefore, it is helpful before submitting code to run the check yourself: +[Continuous Integration](https://github.com/tangyoha/telegram_media_downloader/actions) using GitHub Actions will run those tools and report any stylistic errors in your code. Therefore, it is helpful before submitting code to run the check yourself: ```sh black media_downloader.py utils ``` to auto-format your code. Additionally, many editors have plugins that will apply `black` as you edit files. -Writing good code is not just about what you write. It is also about _how_ you write it. During [Continuous Integration](https://github.com/Dineshkarthik/telegram_media_downloader/actions) testing, several tools will be run to check your code for stylistic errors. Generating any warnings will cause the test to fail. Thus, good style is a requirement for submitting code to telegram-media-downloader. +Writing good code is not just about what you write. It is also about _how_ you write it. During [Continuous Integration](https://github.com/tangyoha/telegram_media_downloader/actions) testing, several tools will be run to check your code for stylistic errors. Generating any warnings will cause the test to fail. Thus, good style is a requirement for submitting code to telegram-media-downloader. This is already added in the repo to help contributors verify their changes before contributing them to the project: ```sh @@ -196,4 +196,4 @@ Explain the motivation for the change in the commit message body. This commit me ### Code of Conduct -As a contributor, you can help us keep the community open and inclusive. Please read and follow our [Code of Conduct](https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CODE_OF_CONDUCT.md). +As a contributor, you can help us keep the community open and inclusive. Please read and follow our [Code of Conduct](https://github.com/tangyoha/telegram_media_downloader/blob/master/CODE_OF_CONDUCT.md). diff --git a/README.md b/README.md index c7d0e7a2..82ef0388 100644 --- a/README.md +++ b/README.md @@ -2,48 +2,55 @@

Telegram Media Downloader

-Unittest +Unittest Coverage Status -License: MIT +License: MIT Code style: black

- Feature request + 中文 · + Feature request · - Report a bug + Report a bug · - Support: Discussions + Support: Discussions & Telegram Community

-### Overview: +### Overview + Download all media files from a conversation or a channel that you are a part of from telegram. A meta of last read/downloaded message is stored in the config file so that in such a way it won't download the same media file again. -### Support: +### Support + | Category | Support | |--|--| -|Language | `Python 3.7 ` and above| +|Language | `Python 3.7` and above| |Download media types| audio, document, photo, video, video_note, voice| -### ToDo: +### ToDo + - Add support for multiple channels/chats. ### Installation For *nix os distributions with `make` availability + ```sh -$ git clone https://github.com/Dineshkarthik/telegram_media_downloader.git -$ cd telegram_media_downloader -$ make install +git clone https://github.com/tangyoha/telegram_media_downloader.git +cd telegram_media_downloader +make install ``` + For Windows which doesn't have `make` inbuilt + ```sh -$ git clone https://github.com/Dineshkarthik/telegram_media_downloader.git -$ cd telegram_media_downloader -$ pip3 install -r requirements.txt +git clone https://github.com/tangyoha/telegram_media_downloader.git +cd telegram_media_downloader +pip3 install -r requirements.txt ``` ## Configuration @@ -52,31 +59,33 @@ All the configurations are passed to the Telegram Media Downloader via `config. **Getting your API Keys:** The very first step requires you to obtain a valid Telegram API key (API id/hash pair): -1. Visit [https://my.telegram.org/apps](https://my.telegram.org/apps) and log in with your Telegram Account. -2. Fill out the form to register a new Telegram application. -3. Done! The API key consists of two parts: **api_id** and **api_hash**. +1. Visit [https://my.telegram.org/apps](https://my.telegram.org/apps) and log in with your Telegram Account. +2. Fill out the form to register a new Telegram application. +3. Done! The API key consists of two parts: **api_id** and **api_hash**. **Getting chat id:** **1. Using web telegram:** -1. Open https://web.telegram.org/?legacy=1#/im -2. Now go to the chat/channel and you will see the URL as something like - - `https://web.telegram.org/?legacy=1#/im?p=u853521067_2449618633394` here `853521067` is the chat id. - - `https://web.telegram.org/?legacy=1#/im?p=@somename` here `somename` is the chat id. - - `https://web.telegram.org/?legacy=1#/im?p=s1301254321_6925449697188775560` here take `1301254321` and add `-100` to the start of the id => `-1001301254321`. - - `https://web.telegram.org/?legacy=1#/im?p=c1301254321_6925449697188775560` here take `1301254321` and add `-100` to the start of the id => `-1001301254321`. +1. Open + +2. Now go to the chat/channel and you will see the URL as something like + - `https://web.telegram.org/?legacy=1#/im?p=u853521067_2449618633394` here `853521067` is the chat id. + - `https://web.telegram.org/?legacy=1#/im?p=@somename` here `somename` is the chat id. + - `https://web.telegram.org/?legacy=1#/im?p=s1301254321_6925449697188775560` here take `1301254321` and add `-100` to the start of the id => `-1001301254321`. + - `https://web.telegram.org/?legacy=1#/im?p=c1301254321_6925449697188775560` here take `1301254321` and add `-100` to the start of the id => `-1001301254321`. **2. Using bot:** + 1. Use [@username_to_id_bot](https://t.me/username_to_id_bot) to get the chat_id of - almost any telegram user: send username to the bot or just forward their message to the bot - any chat: send chat username or copy and send its joinchat link to the bot - public or private channel: same as chats, just copy and send to the bot - id of any telegram bot - ### config.yaml + ```yaml api_hash: your_api_hash api_id: your_api_id @@ -98,27 +107,42 @@ file_formats: video: - mp4 save_path: D:\telegram_media_downloader +file_path_prefix: +- chat_title +- media_datetime disable_syslog: - INFO ``` -- api_hash - The api_hash you got from telegram apps -- api_id - The api_id you got from telegram apps -- chat_id - The id of the chat/channel you want to download media. Which you get from the above-mentioned steps. -- last_read_message_id - If it is the first time you are going to read the channel let it be `0` or if you have already used this script to download media it will have some numbers which are auto-updated after the scripts successful execution. Don't change it. -- ids_to_retry - `Leave it as it is.` This is used by the downloader script to keep track of all skipped downloads so that it can be downloaded during the next execution of the script. -- media_types - Type of media to download, you can update which type of media you want to download it can be one or any of the available types. -- file_formats - File types to download for supported media types which are `audio`, `document` and `video`. Default format is `all`, downloads all files. -- save_path - Where you want to storge download file -- disable_syslog - You can choose which types of logs to disable,see `logging._nameToLevel` +- **api_hash** - The api_hash you got from telegram apps +- **api_id** - The api_id you got from telegram apps +- **chat_id** - The id of the chat/channel you want to download media. Which you get from the above-mentioned steps. +- **last_read_message_id** - If it is the first time you are going to read the channel let it be `0` or if you have already used this script to download media it will have some numbers which are auto-updated after the scripts successful execution. Don't change it. +- **ids_to_retry** - `Leave it as it is.` This is used by the downloader script to keep track of all skipped downloads so that it can be downloaded during the next execution of the script. +- **media_types** - Type of media to download, you can update which type of media you want to download it can be one or any of the available types. +- **file_formats** - File types to download for supported media types which are `audio`, `document` and `video`. Default format is `all`, downloads all files. +- **save_path** - The root directory where you want to store downloaded files. +- **file_path_prefix** - Store file subfolders, the order of the list is not fixed, can be randomly combined. + - `chat_title` - channel or group title, it will be chat id if not exist title. + - `media_datetime` - media date, also see pyrogram.types.Message.date.strftime("%Y_%m"). + - `meida_type` - meida type, also see `media_types`. +- **disable_syslog** - You can choose which types of logs to disable,see `logging._nameToLevel`. + ## Execution + ```sh -$ python3 media_downloader.py +python3 media_downloader.py ``` -All the downloaded media will be stored inside `save_path`, and floder names include channel id or chat id.All files are rented by `month`. +All downloaded media will be stored at the root of `save_path`. +The specific location reference is as follows: + +The complete directory of video download is: `save_path`/`chat_title`/`media_datetime`/`meida_type`. +The order of the list is not fixed and can be randomly combined. +If the configuration is empty, all files are saved under `save_path`. ## Proxy + `socks4, socks5, http` proxies are supported in this project currently. To use it, add the following to the bottom of your `config.yaml` file ```yaml @@ -129,14 +153,19 @@ proxy: username: your_username password: your_password ``` + If your proxy doesn’t require authorization you can omit username and password. Then the proxy will automatically be enabled. ## Contributing + ### Contributing Guidelines -Read through our [contributing guidelines](https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CONTRIBUTING.md) to learn about our submission process, coding rules and more. + +Read through our [contributing guidelines](https://github.com/tangyoha/telegram_media_downloader/blob/master/CONTRIBUTING.md) to learn about our submission process, coding rules and more. ### Want to Help? -Want to file a bug, contribute some code, or improve documentation? Excellent! Read up on our guidelines for [contributing](https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CONTRIBUTING.md). + +Want to file a bug, contribute some code, or improve documentation? Excellent! Read up on our guidelines for [contributing](https://github.com/tangyoha/telegram_media_downloader/blob/master/CONTRIBUTING.md). ### Code of Conduct -Help us keep Telegram Media Downloader open and inclusive. Please read and follow our [Code of Conduct](https://github.com/Dineshkarthik/telegram_media_downloader/blob/master/CODE_OF_CONDUCT.md). + +Help us keep Telegram Media Downloader open and inclusive. Please read and follow our [Code of Conduct](https://github.com/tangyoha/telegram_media_downloader/blob/master/CODE_OF_CONDUCT.md). diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 00000000..2fb9b7cd --- /dev/null +++ b/README_CN.md @@ -0,0 +1,179 @@ + +

电报资源下载

+ +

+Unittest +Coverage Status +License: MIT +Code style: black +

+ +

+ English · + 新功能请求 + · + 报告bug + · + 帮助: 讨论 + & + 电报讨论群 +

+ +### 概述 + +从您所属的电报对话或频道下载所有媒体文件。 +最后读取/下载消息的元数据存储在配置文件中，这样它就不会再次下载相同的媒体文件。 + +### 支持 + +| 类别 | 支持 | +| ------------ | ---------------------------------------- | +| 语言 | `Python 3.7` 及以上 | +| 下载媒体类型 | 音频、文档、照片、视频、video_note、语音 | + +### 待做 + +- 添加对多个频道/聊天的支持。 + +### 安装 + +对于具有 `make` 可用性的 *nix 操作系统发行版 + +```sh +git clone https://github.com/tangyoha/telegram_media_downloader.git +cd telegram_media_downloader +make install +``` + +对于没有内置 `make` 的 Windows + +```sh +git clone https://github.com/tangyoha/telegram_media_downloader.git +cd telegram_media_downloader +pip3 install -r requirements.txt +``` + +## 配置 + +所有配置都通过 config.yaml 文件传递​​给 `Telegram Media Downloader`。 + +**获取您的 API 密钥：** +第一步需要您获得有效的 Telegram API 密钥（API id/hash pair）： + +1. 访问 [https://my.telegram.org/apps](https://my.telegram.org/apps) 并使用您的 Telegram 帐户登录。 +2. 填写表格以注册新的 Telegram 应用程序。 +3. 完成！ API 密钥由两部分组成：**api_id** 和**api_hash**。 + +**获取聊天ID：** +> 如果你需要下载收藏夹的内容请填`me` + +**1。使用网络电报：** + +1. 打开 +2. 现在转到聊天/频道，您将看到 URL 类似 + +- `https://web.telegram.org/?legacy=1#/im?p=u853521067_2449618633394` 这里 `853521067` 是聊天 ID。 +- `https://web.telegram.org/?legacy=1#/im?p=@somename` 这里的 `somename` 是聊天 ID。 +- `https://web.telegram.org/?legacy=1#/im?p=s1301254321_6925449697188775560` 此处取 `1301254321` 并将 `-100` 添加到 id => `-1001301254321` 的开头。 +- `https://web.telegram.org/?legacy=1#/im?p=c1301254321_6925449697188775560` 此处取 `1301254321` 并将 `-100` 添加到 id => `-1001301254321` 的开头。 + +**2。使用机器人：** +1.使用[@username_to_id_bot](https://t.me/username_to_id_bot)获取chat_id + - 几乎所有电报用户：将用户名发送给机器人或将他们的消息转发给机器人 + - 任何聊天：发送聊天用户名或复制并发送其加入聊天链接到机器人 + - 公共或私人频道：与聊天相同，只需复制并发送给机器人 + - 任何电报机器人的 ID + +### 配置文件 + +```yaml +api_hash: your_api_hash +api_id: your_api_id +chat_id: telegram_chat_id +last_read_message_id: 0 +ids_to_retry: [] +media_types: +- audio +- document +- photo +- video +- voice +file_formats: + audio: + - all + document: + - pdf + - epub + video: + - mp4 +save_path: D:\telegram_media_downloader +group_by_media_type: true +file_path_prefix: +- chat_title +- media_datetime +disable_syslog: +- INFO +``` + +- **api_hash** - 你从电报应用程序获得的 api_hash +- **api_id** - 您从电报应用程序获得的 api_id +- **chat_id** - 您要下载媒体的聊天/频道的 ID。你从上述步骤中得到的。 +- **last_read_message_id** - 如果这是您第一次阅读频道，请将其设置为“0”，或者如果您已经使用此脚本下载媒体，它将有一些数字，这些数字会在脚本成功执行后自动更新。不要改变它。如果你需要下载收藏夹的内容，请填`me`。 +- **ids_to_retry** - `保持原样。`下载器脚本使用它来跟踪所有跳过的下载，以便在下次执行脚本时可以下载它。 +- **media_types** - 要下载的媒体类型，您可以更新要下载的媒体类型，它可以是一种或任何可用类型。 +- **file_formats** - 为支持的媒体类型（“音频”、“文档”和“视频”）下载的文件类型。默认格式为“all”，下载所有文件。 +- **save_path** - 你想存储下载文件的根目录 +- **file_path_prefix** - 存储文件子文件夹，列表的顺序不定，可以随机组合 + - `chat_title` - 聊天频道或者群组标题, 如果找不到标题则为配置文件中的`chat_id` + - `media_datetime` - 资源的发布时间 + - `meida_type` - 资源类型，类型查阅 `media_types` +- **disable_syslog** - 您可以选择要禁用的日志类型，请参阅 `logging._nameToLevel` + +## 执行 + +```sh +python3 media_downloader.py +``` + +所有下载的媒体都将存储在`save_path`根目录下。 +具体位置参考如下： + +```yaml +file_path_prefix: + - chat_title + - media_datetime + - meida_type +``` + +视频下载完整目录为：`save_path`/`chat_title`/`media_datetime`/`meida_type`。 +列表的顺序不定，可以随机组合。 +如果配置为空，则所有文件保存在`save_path`下。 + +## 代理 + +该项目目前支持 socks4、socks5、http 代理。要使用它，请将以下内容添加到`config.yaml`文件的底部 + +```yaml +代理人： + 方案：socks5 + 主机名：11.22.33.44 + 端口：1234 + 用户名：你的用户名 + 密码：你的密码 +``` + +如果您的代理不需要授权，您可以省略用户名和密码。然后代理将自动启用。 + +## 贡献 + +### 贡献指南 + +通读我们的[贡献指南](./CONTRIBUTING.md)，了解我们的提交流程、编码规则等。 + +### 想帮忙？ + +想要提交错误、贡献一些代码或改进文档？出色的！阅读我们的 [贡献] 指南 (./CONTRIBUTING.md)。 + +### 行为守则 + +帮助我们保持 Telegram Media Downloader 的开放性和包容性。请阅读并遵守我们的[行为准则](./CODE_OF_CONDUCT.md)。 diff --git a/config.yaml b/config.yaml index 2f8bd867..b1143a1f 100644 --- a/config.yaml +++ b/config.yaml @@ -1,8 +1,22 @@ -api_hash: your_api_hash -api_id: your_api_id -chat_id: telegram_chat_id -last_read_message_id: 0 +api_hash: f689e50c58309525f591e53dbf20b325 +api_id: 10502402 +bot_token: 5331758252:AAG05jKtjEQFjnNlMeeArefFxilAMDZddt8 +chat_id: me +disable_syslog: +- INFO +file_formats: + audio: + - all + document: + - all + video: + - all +file_path_prefix: +- chat_title +- media_datetime + ids_to_retry: [] +last_read_message_id: 0 media_types: - audio - photo @@ -10,12 +24,8 @@ media_types: - document - voice - video_note -file_formats: - audio: - - all - document: - - all - video: - - all -disable_syslog: -- INFO \ No newline at end of file +proxy: + hostname: 127.0.0.1 + port: 10808 + scheme: socks5 +save_path: E:\github\telegram_media_downloader diff --git a/media_downloader.py b/media_downloader.py index 585086e6..f93d9f0e 100644 --- a/media_downloader.py +++ b/media_downloader.py @@ -54,16 +54,15 @@ def _check_download_finish(media_size: Any, download_path: str, message_id: int) if media_size is not None: download_size = os.path.getsize(download_path) if media_size == download_size: - logger.log(CUSTOM_LOG, - "Media downloaded - %s", download_path) + logger.log(CUSTOM_LOG, "Media downloaded - %s", download_path) app.downloaded_ids.append(message_id) app.total_download_task += 1 else: - logger.log(CUSTOM_LOG, - 'Media downloaded with wrong size - %s', download_path) + logger.log( + CUSTOM_LOG, "Media downloaded with wrong size - %s", download_path + ) os.remove(download_path) - raise TypeError( - 'Media downloaded with wrong size') + raise TypeError("Media downloaded with wrong size") def _validate_title(title: str): @@ -146,14 +145,13 @@ async def _get_media_meta( """ if _type in ["audio", "document", "video"]: # pylint: disable = C0301 - file_format: Optional[str] = media_obj.mime_type.split( # type: ignore - "/")[-1] + file_format: Optional[str] = media_obj.mime_type.split("/")[-1] # type: ignore else: file_format = None - dirname = _validate_title(f'{app.chat_id}') - if message.chat: - dirname = _validate_title(f'{message.chat.title}') + dirname = _validate_title(f"{app.chat_id}") + if message.chat and message.chat.title: + dirname = _validate_title(f"{message.chat.title}") if message.date: datetime_dir_name = message.date.strftime("%Y_%m") @@ -163,8 +161,10 @@ async def _get_media_meta( if _type in ["voice", "video_note"]: # pylint: disable = C0209 file_format = media_obj.mime_type.split("/")[-1] # type: ignore + file_save_path = app.get_file_save_path(_type, dirname, datetime_dir_name) + file_name: str = os.path.join( - app.save_path, dirname, datetime_dir_name, + file_save_path, "{} - {}_{}.{}".format( message.id, _type, @@ -180,8 +180,8 @@ async def _get_media_meta( file_format = "jpg" file_name = f"{file_name}.{file_format}" - file_name = os.path.join( - app.save_path, dirname, datetime_dir_name, f"{message.id} - {file_name}") + file_save_path = app.get_file_save_path(_type, dirname, datetime_dir_name) + file_name = os.path.join(file_save_path, f"{message.id} - {file_name}") return file_name, file_format @@ -239,8 +239,11 @@ async def download_media( # if media_size is not None and file_size != media_size: # FIXME: if exist and not empty file skip - logger.log(CUSTOM_LOG, - "%s alreay download,download skipped.\n", file_name) + logger.log( + CUSTOM_LOG, + "%s alreay download,download skipped.\n", + file_name, + ) break download_path = await client.download_media( @@ -248,10 +251,9 @@ async def download_media( ) if download_path and isinstance(download_path, str): - media_size = getattr(_media, 'file_size') + media_size = getattr(_media, "file_size") # TODO: if not exist file size or media - _check_download_finish( - media_size, download_path, message.id) + _check_download_finish(media_size, download_path, message.id) app.downloaded_ids.append(message.id) break @@ -362,6 +364,8 @@ async def begin_import(pagination_limit: int): proxy=app.proxy, ) await client.start() + print("Successfully started (Press Ctrl+C to stop)") + last_read_message_id: int = app.last_read_message_id messages_iter = client.get_chat_history( app.chat_id, offset_id=app.last_read_message_id, reverse=True @@ -391,7 +395,9 @@ async def begin_import(pagination_limit: int): app.update_config() async for message in messages_iter: # type: ignore - if pagination_count != pagination_limit and not app.need_skip_message(message.id): + if pagination_count != pagination_limit and not app.need_skip_message( + message.id + ): pagination_count += 1 messages_list.append(message) else: @@ -421,26 +427,30 @@ async def begin_import(pagination_limit: int): def main(): """Main function of the downloader.""" try: - asyncio.get_event_loop().run_until_complete( - begin_import(pagination_limit=100) - ) + asyncio.get_event_loop().run_until_complete(begin_import(pagination_limit=100)) if app.failed_ids: - logger.log(CUSTOM_LOG, - "Downloading of %d files failed. " - "Failed message ids are added to config file.\n" - "These files will be downloaded on the next run.", - len(set(app.failed_ids)), - ) - app.update_config() + logger.log( + CUSTOM_LOG, + "Downloading of %d files failed. " + "Failed message ids are added to config file.\n" + "These files will be downloaded on the next run.", + len(set(app.failed_ids)), + ) check_for_updates() + except KeyboardInterrupt: + logger.log(CUSTOM_LOG, "Stopped!") except Exception as e: logger.exception("%s", e) + finally: + logger.log(CUSTOM_LOG, "update config......") app.update_config() if __name__ == "__main__": print_meta(logger) main() - logger.log(CUSTOM_LOG, - "Updated last read message_id to config file, total download %s", - app.total_download_task) + logger.log( + CUSTOM_LOG, + "Updated last read message_id to config file, total download %s", + app.total_download_task, + ) diff --git a/module/app.py b/module/app.py index 898e3a75..d785ec2f 100644 --- a/module/app.py +++ b/module/app.py @@ -1,6 +1,7 @@ """Application module""" import os from typing import List + import yaml # pylint: disable = R0902 @@ -23,7 +24,7 @@ def __init__(self, config_file: str): self.reset() try: - with open(os.path.join(os.path.abspath('.'), self.config_file)) as f: + with open(os.path.join(os.path.abspath("."), self.config_file)) as f: self.config = yaml.safe_load(f) self.load_config(self.config) except Exception: @@ -36,7 +37,7 @@ def reset(self): self.downloaded_ids: list = [] self.failed_ids: list = [] self.disable_syslog: list = [] - self.save_path = os.path.abspath('.') + self.save_path = os.path.abspath(".") self.ids_to_retry: list = [] self.api_id: str = "" self.api_hash: str = "" @@ -47,6 +48,7 @@ def reset(self): self.last_read_message_id = 0 self.restart_program = False self.config: dict = {} + self.file_path_prefix: List[str] = ["chat_title", "media_datetime"] def load_config(self, _config: dict) -> bool: """load config from str. @@ -87,9 +89,42 @@ def load_config(self, _config: dict) -> bool: self.proxy = _config["proxy"] if _config.get("restart_program"): self.restart_program = _config["restart_program"] - + if _config.get("file_path_prefix"): + self.file_path_prefix = _config["file_path_prefix"] return True + def get_file_save_path( + self, media_type: str, chat_title: str, media_datetime: str + ) -> str: + """Get file save path prefix. + + Parameters + ---------- + media_type: str + see config.yaml media_types + + chat_title: str + see channel or group title + + media_datetime: str + media datatime + + Returns + ------- + str + file save path prefix + """ + + res: str = self.save_path + for iter in self.file_path_prefix: + if iter == "chat_title": + res = os.path.join(res, chat_title) + elif iter == "media_datetime": + res = os.path.join(res, media_datetime) + elif iter == "media_type": + res = os.path.join(res, media_type) + return res + def need_skip_message(self, message_id: int) -> bool: """if need skip download message. @@ -115,14 +150,14 @@ def update_config(self, immediate: bool = True): # pylint: disable = W0201 self.ids_to_retry = ( - list(set(self.ids_to_retry) - - set(self.downloaded_ids)) + self.failed_ids + list(set(self.ids_to_retry) - set(self.downloaded_ids)) + self.failed_ids ) self.config["last_read_message_id"] = self.last_read_message_id self.config["ids_to_retry"] = self.ids_to_retry self.config["disable_syslog"] = self.disable_syslog self.config["save_path"] = self.save_path + self.config["file_path_prefix"] = self.file_path_prefix if immediate: with open(self.config_file, "w") as yaml_file: diff --git a/requirements.txt b/requirements.txt index 7cad49df..31b7eb90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -https://github.com/Dineshkarthik/pyrogram/archive/refs/heads/master.zip +https://github.com/tangyoha/pyrogram/archive/refs/heads/master.zip PyYAML==6.0 rich==12.5.1 TgCrypto==1.2.3 diff --git a/setup.py b/setup.py index 717a3bbc..ffcec245 100644 --- a/setup.py +++ b/setup.py @@ -5,11 +5,11 @@ setup( name="telegram-media-downloader", version=__version__, - author="Dineshkarthik Raveendran", - author_email="hello@dineshkarthik.me", + author="tangyoha", + author_email="tangyoha@outlook.com", description="A simple script to download media from telegram", - url="https://github.com/Dineshkarthik/telegram_media_downloader", - download_url="https://github.com/Dineshkarthik/telegram_media_downloader/releases/latest", + url="https://github.com/tangyoha/telegram_media_downloader", + download_url="https://github.com/tangyoha/telegram_media_downloader/releases/latest", py_modules=["media_downloader"], classifiers=[ "Development Status :: 5 - Production/Stable", @@ -34,9 +34,9 @@ "Topic :: Software Development :: Libraries :: Python Modules", ], project_urls={ - "Tracker": "https://github.com/Dineshkarthik/telegram_media_downloader/issues", + "Tracker": "https://github.com/tangyoha/telegram_media_downloader/issues", "Community": "https://t.me/tgmdnews", - "Source": "https://github.com/Dineshkarthik/telegram_media_downloader", + "Source": "https://github.com/tangyoha/telegram_media_downloader", }, python_requires="~=3.7", ) diff --git a/tests/test_app/test_app.py b/tests/test_app/test_app.py index abbe42af..a35bbda4 100644 --- a/tests/test_app/test_app.py +++ b/tests/test_app/test_app.py @@ -1,13 +1,11 @@ -from module.app import Application import os import sys import unittest from unittest import mock + import mock -from module.app import ( - Application -) +from module.app import Application sys.path.append("..") # Adds higher directory to python modules path. @@ -15,13 +13,13 @@ class AppTestCase(unittest.TestCase): @classmethod def tearDownClass(cls): - config_test = os.path.join(os.path.abspath('.'), "config_test.yaml") + config_test = os.path.join(os.path.abspath("."), "config_test.yaml") if os.path.exists(config_test): os.remove(config_test) def test_app(self): app = Application("") - self.assertEqual(app.save_path, os.path.abspath('.')) + self.assertEqual(app.save_path, os.path.abspath(".")) self.assertEqual(app.proxy, {}) self.assertEqual(app.restart_program, False) @@ -31,8 +29,7 @@ def test_app(self): app.update_config(False) - self.assertEqual(app.last_read_message_id, - app.config["last_read_message_id"]) + self.assertEqual(app.last_read_message_id, app.config["last_read_message_id"]) self.assertEqual(app.ids_to_retry, app.config["ids_to_retry"]) @mock.patch("__main__.__builtins__.open", new_callable=mock.mock_open) @@ -43,5 +40,6 @@ def test_update_config(self, mock_yaml, mock_open): app.update_config() mock_open.assert_called_with("config_test.yaml", "w") mock_yaml.dump.assert_called_with( - app.config, mock.ANY, default_flow_style=False) + app.config, mock.ANY, default_flow_style=False + ) mock_open.assert_called_with("config_test.yaml", "w") diff --git a/tests/test_media_downloader.py b/tests/test_media_downloader.py index 43652c31..b10471c2 100644 --- a/tests/test_media_downloader.py +++ b/tests/test_media_downloader.py @@ -12,11 +12,11 @@ _can_download, _get_media_meta, _is_exist, + app, begin_import, download_media, main, process_messages, - app, ) MOCK_DIR: str = "/root/project" @@ -30,7 +30,7 @@ "ids_to_retry": [1, 2], "media_types": ["audio", "voice"], "file_formats": {"audio": ["all"], "voice": ["all"]}, - "save_path": MOCK_DIR + "save_path": MOCK_DIR, } @@ -98,8 +98,9 @@ def __init__(self, **kwargs): self.voice = kwargs.get("voice", None) self.video_note = kwargs.get("video_note", None) if kwargs.get("dis_chat") == None: - self.chat = Chat(kwargs.get("chat_id", None), - kwargs.get("chat_title", None)) + self.chat = Chat( + kwargs.get("chat_id", None), kwargs.get("chat_title", None) + ) else: self.chat = None self.date: datetime = None @@ -293,7 +294,7 @@ async def get_messages(self, *args, **kwargs): file_name="sample_video2.mov", mime_type="video/mov", ), - ) + ), ] return [] @@ -348,16 +349,16 @@ def test_get_media_meta(self): media=True, date=datetime(2019, 8, 5, 14, 35, 12), chat_title="test2", - photo=MockPhoto(date=datetime(2019, 8, 5, 14, 35, - 12), file_unique_id="ADAVKJYIFV"), + photo=MockPhoto( + date=datetime(2019, 8, 5, 14, 35, 12), file_unique_id="ADAVKJYIFV" + ), ) result = self.loop.run_until_complete( async_get_media_meta(message, message.photo, "photo") ) self.assertEqual( ( - platform_generic_path( - "/root/project/test2/2019_08/2 - ADAVKJYIFV.jpg"), + platform_generic_path("/root/project/test2/2019_08/2 - ADAVKJYIFV.jpg"), "jpg", ), result, @@ -378,8 +379,7 @@ def test_get_media_meta(self): ) self.assertEqual( ( - platform_generic_path( - "/root/project/test2/0/3 - sample_document.pdf"), + platform_generic_path("/root/project/test2/0/3 - sample_document.pdf"), "pdf", ), result, @@ -402,7 +402,8 @@ def test_get_media_meta(self): self.assertEqual( ( platform_generic_path( - "/root/project/test2/2021_08/4 - sample_audio.mp3"), + "/root/project/test2/2021_08/4 - sample_audio.mp3" + ), "mp3", ), result, @@ -423,8 +424,7 @@ def test_get_media_meta(self): ) self.assertEqual( ( - platform_generic_path( - "/root/project/test2/2022_08/5 - None.mp4"), + platform_generic_path("/root/project/test2/2022_08/5 - None.mp4"), "mp4", ), result, @@ -446,8 +446,7 @@ def test_get_media_meta(self): ) self.assertEqual( ( - platform_generic_path( - "/root/project/test2/2022_08/5 - test.mp4"), + platform_generic_path("/root/project/test2/2022_08/5 - test.mp4"), "mp4", ), result, @@ -471,8 +470,7 @@ def test_get_media_meta(self): print(app.chat_id) self.assertEqual( ( - platform_generic_path( - "/root/project/8654123/2022_08/5 - test.mp4"), + platform_generic_path("/root/project/8654123/2022_08/5 - test.mp4"), "mp4", ), result, @@ -723,7 +721,7 @@ def test_issues_311(self): ), ) - media_size = getattr(message.video, 'file_size') + media_size = getattr(message.video, "file_size") self.assertEqual(media_size, 1024) self.loop.run_until_complete( diff --git a/tests/utils/test_updates.py b/tests/utils/test_updates.py index d04c2d30..1df354b0 100644 --- a/tests/utils/test_updates.py +++ b/tests/utils/test_updates.py @@ -27,7 +27,7 @@ def __init__(self, status): def read(self): if self.status == 200: - return b'{"name":"v0.0.0 2022-03-02","tag_name":"v0.0.0", "html_url":"https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0"}' + return b'{"name":"v0.0.0 2022-03-02","tag_name":"v0.0.0", "html_url":"https://github.com/tangyoha/telegram_media_downloader/releases/tag/v0.0.0"}' else: return b"{error}" @@ -43,7 +43,9 @@ class UpdatesTestCase(unittest.TestCase): def test_update(self, mock_markdown, mock_console): check_for_updates() name: str = "v0.0.0 2022-03-02" - html_url: str = "https://github.com/Dineshkarthik/telegram_media_downloader/releases/tag/v0.0.0" + html_url: str = ( + "https://github.com/tangyoha/telegram_media_downloader/releases/tag/v0.0.0" + ) expected_message: str = ( f"## New version of Telegram-Media-Downloader is available - {name}\n" "You are using an outdated version v0.0.1 please pull in the changes using `git pull` or download the latest release.\n\n" diff --git a/utils/__init__.py b/utils/__init__.py index 400808d9..5d3115a1 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -1,5 +1,5 @@ """Init namespace""" -__version__ = "2.0.2" +__version__ = "2.1.0" __license__ = "MIT License" -__copyright__ = "Copyright (C) 2019 Dineshkarthik " +__copyright__ = "Copyright (C) 2022 tangyoha "