main.yml

name: Download NammaYatri Stats

on:
  workflow_dispatch:
  push:
    branches: ["main"]
    paths: ["DataURLs.txt"]
  schedule:
    - cron:  '29 18 * * *'

permissions:
  contents: write
  pull-requests: write

jobs:
  download-json-files:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v4.1.1
      
    - name: Download JSON files
      run: |
        mkdir -p raw-data/opendata
        cat DataURLs.txt | parallel -j 4 --progress --joblog download.log '
          url={1}
          clean_url=$(echo "$url" | sed "s#https://d11gklsvr97l1g.cloudfront.net/open/json-data/#opendata/#")
          filename=$(basename "$clean_url")
          folder=$(dirname "$clean_url")
          mkdir -p "raw-data/$folder"
          curl -L "$url" -o "raw-data/$folder/$filename"
        '
    

    - name: Extract Reviews
      run: |
        mkdir -p raw-data/reviews
        cd script
        python ExtractReviews.py
        
    - name: Commit and push changes
      uses: stefanzweifel/git-auto-commit-action@v5
      with:
        commit_message: "Add new JSON files"
        commit_options: '--no-verify'
        file_pattern: 'raw-data/*.json raw-data/reviews/*.json APP*.md'