Merge pull request #380 from hummingbot/core

test/core-connectors

.github/workflows/workflow.yml

@@ -78,17 +78,15 @@ jobs:
           mkdir conf db
           cp -rf src/templates/* conf
           sed -i 's|/home/gateway/conf/lists/|conf/lists/|g' ./conf/*.yml
-          sed -i 's/https:\/\/rpc.ankr.com\/eth_goerli/http:\/\/127.0.0.1:8545\//g' ./conf/ethereum.yml
-          sed -i 's/https:\/\/etc.rivet.link/http:\/\/127.0.0.1:8545\//g' ./conf/ethereum-classic.yml
+          sed -i 's/https:\/\/rpc.ankr.com\/eth_sepolia/http:\/\/127.0.0.1:8545\//g' ./conf/ethereum.yml
 
       - name: Run unit test coverage
         if: github.event_name == 'pull_request'
         shell: bash
         run: |
           git fetch --all -q
           git checkout -b $GITHUB_SHA
-          NODE_OPTIONS=--max-old-space-size=10240 node ./node_modules/.bin/jest --runInBand --coverage ./test/
-#          git diff origin/$GITHUB_BASE_REF | yarn diff-test-coverage -c ./coverage/lcov.info -t lcov -b 50 -l 60
+          GATEWAY_TEST_MODE=dev ./node_modules/.bin/jest --runInBand --coverage ./test/
 
   docker_build_and_push:
     runs-on: ubuntu-latest

.gitignore

@@ -17,4 +17,7 @@ package-lock.json
 .history
 
 # Yarn
-yarn-error.log
+yarn-error.log
+
+# Larp
+.env

README.md

@@ -71,7 +71,7 @@ There are a number of ways to contribute to gateway.
 
 ### Configuration
 
-- If you want to turn off `https`, set `unsafeDevModeWithHTTP` to `true` in [conf/server.yml](./conf/server.yml). 
+- To run in HTTP mode (for development), use `yarn start --dev`. By default, Gateway runs in secure HTTPS mode.
 
 - If you want Gateway to log to standard out, set `logToStdOut` to `true` in [conf/server.yml](./conf/server.yml).
 
@@ -131,3 +131,4 @@ You can run the prettifier before committing with:
 yarn run prettier
 ```
 
+

add-token.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+prompt_proceed() {
+  read -p "Do you want to proceed? [Y/N] >>> " PROCEED
+  if [ "$PROCEED" == "" ]; then
+    prompt_proceed
+  else
+    if [[ "$PROCEED" != "Y" && "$PROCEED" != "y" ]]; then
+      PROCEED="N"
+    fi
+  fi
+}
+
+validate_address() {
+  if [[ ! "$1" =~ ^0x[a-fA-F0-9]{40}$ ]]; then
+    return 1
+  fi
+  return 0
+}
+
+validate_decimals() {
+  if [[ ! "$1" =~ ^[0-9]+$ ]] || [ "$1" -lt 1 ] || [ "$1" -gt 18 ]; then
+    return 1
+  fi
+  return 0
+}
+
+add_token() {
+  INPUT_PATH=$1
+
+  # Check if input file exists
+  if [ ! -f "$INPUT_PATH" ]; then
+    echo "Error: Input file $INPUT_PATH does not exist"
+    exit 1
+  fi
+
+  # Get chainId from existing tokens
+  CHAIN_ID=$(jq -r '.[0].chainId' "$INPUT_PATH")
+  if [ -z "$CHAIN_ID" ] || [ "$CHAIN_ID" = "null" ]; then
+    echo "Error: Could not determine chainId from existing tokens"
+    exit 1
+  fi
+
+  # Prompt for token details
+  while true; do
+    read -p "Enter token address (0x...) >>> " ADDRESS
+    if validate_address "$ADDRESS"; then
+      # Check for duplicate address
+      if jq -e --arg addr "$ADDRESS" '.[] | select(.address == $addr)' "$INPUT_PATH" > /dev/null; then
+        echo "Error: Token with this address already exists"
+        continue
+      fi
+      break
+    else
+      echo "Error: Invalid address format. Must be 0x followed by 40 hex characters"
+    fi
+  done
+
+  while true; do
+    read -p "Enter token symbol >>> " SYMBOL
+    if [ -z "$SYMBOL" ]; then
+      echo "Error: Symbol is required"
+      continue
+    fi
+    # Check for duplicate symbol
+    if jq -e --arg sym "$SYMBOL" '.[] | select(.symbol == $sym)' "$INPUT_PATH" > /dev/null; then
+      echo "Error: Token with this symbol already exists"
+      continue
+    fi
+    break
+  done
+
+  read -p "Enter token name (press Enter to use symbol) >>> " NAME
+  NAME=${NAME:-$SYMBOL}
+
+  while true; do
+    read -p "Enter token decimals (1-18) >>> " DECIMALS
+    if validate_decimals "$DECIMALS"; then
+      break
+    else
+      echo "Error: Decimals must be a number between 1 and 18"
+    fi
+  done
+
+  # Create new token JSON
+  NEW_TOKEN=$(cat <<EOF
+{
+  "chainId": $CHAIN_ID,
+  "address": "$ADDRESS",
+  "symbol": "$SYMBOL",
+  "name": "$NAME",
+  "decimals": $DECIMALS
+}
+EOF
+)
+
+  # Show confirmation
+  echo
+  echo "Review new token entry:"
+  echo "$NEW_TOKEN" | jq .
+  echo
+
+  prompt_proceed
+  if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" ]]; then
+    # Add new token to list
+    jq --argjson newToken "$NEW_TOKEN" '. + [$newToken]' "$INPUT_PATH" > "${INPUT_PATH}.tmp"
+    mv "${INPUT_PATH}.tmp" "$INPUT_PATH"
+    echo "Token successfully added to $INPUT_PATH"
+  else
+    echo "Operation cancelled"
+    exit 1
+  fi
+}
+
+# Check if jq is installed
+if ! command -v jq &> /dev/null; then
+  echo "Error: jq is required but not installed."
+  echo "Please install jq first:"
+  echo "  Ubuntu/Debian: sudo apt-get install jq"
+  echo "  MacOS: brew install jq"
+  exit 1
+fi
+
+echo
+echo
+echo "===============  ADD TOKEN TO LIST ==============="
+echo
+echo
+
+# Get input path
+read -p "Enter path to the token list file >>> " INPUT_PATH
+
+add_token "$INPUT_PATH" 

clean-tokenlist.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+# init
+
+SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+TEMPLATE_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/src/templates"
+
+prompt_proceed () {
+  read -p "Do you want to proceed? [Y/N] >>> " PROCEED
+  if [ "$PROCEED" == "" ]
+  then
+    prompt_proceed
+  else
+    if [[ "$PROCEED" != "Y" && "$PROCEED" != "y" ]]
+    then
+      PROCEED="N"
+    fi
+  fi
+}
+
+clean_token_list () {
+  INPUT_PATH=$1
+  CHAIN_ID=$2
+
+  # Create output path with _CLEANED suffix and chainId
+  OUTPUT_PATH="${INPUT_PATH%.*}_CLEANED${CHAIN_ID:+_$CHAIN_ID}.json"
+
+  # Check if input file exists
+  if [ ! -f "$INPUT_PATH" ]; then
+    echo "Error: Input file $INPUT_PATH does not exist"
+    exit 1
+  fi
+
+  # Create output directory if it doesn't exist
+  OUTPUT_DIR=$(dirname "$OUTPUT_PATH")
+  mkdir -p "$OUTPUT_DIR"
+
+  # Use jq to clean the token list and remove duplicates
+  jq --arg chainid "$CHAIN_ID" '
+    (if type == "object" then .tokens else . end) |
+    map({
+      chainId: .chainId,
+      name: .name,
+      symbol: .symbol,
+      address: .address,
+      decimals: .decimals
+    } | select(
+      .chainId != null and
+      ($chainid == "" or .chainId == ($chainid|tonumber)) and
+      .name != null and
+      .symbol != null and
+      .address != null and
+      .decimals != null
+    )) as $all |
+    
+    # Group by symbol and process duplicates
+    ($all | group_by(.symbol) | map(select(length > 1)) | map({
+      symbol: .[0].symbol,
+      kept: .[0],
+      removed: .[1:]
+    })) as $duplicates |
+    
+    # Output cleaned list and duplicate info
+    {
+      "tokens": ($all | unique_by(.symbol)),
+      "duplicates": $duplicates
+    }
+  ' "$INPUT_PATH" > "${OUTPUT_PATH}.tmp"
+
+  # Extract the cleaned token list
+  jq '.tokens' "${OUTPUT_PATH}.tmp" > "$OUTPUT_PATH"
+
+  # Report duplicates if any exist
+  echo "Checking for duplicate symbols..."
+  DUPLICATES=$(jq -r '.duplicates[] | "Symbol: \(.symbol)\n  Kept: \(.kept.address)\n  Removed: \(.removed | map(.address) | join(", "))"' "${OUTPUT_PATH}.tmp")
+  if [ ! -z "$DUPLICATES" ]; then
+    echo "Found tokens with duplicate symbols:"
+    echo "$DUPLICATES"
+  else
+    echo "No duplicate symbols found"
+  fi
+
+  # Remove temporary file
+  rm "${OUTPUT_PATH}.tmp"
+
+  # Check if the operation was successful
+  if [ $? -eq 0 ]; then
+    echo "Token list successfully cleaned and saved to $OUTPUT_PATH"
+  else
+    echo "Error cleaning token list"
+    exit 1
+  fi
+}
+
+# Check if jq is installed
+if ! command -v jq &> /dev/null; then
+  echo "Error: jq is required but not installed."
+  echo "Please install jq first:"
+  echo "  Ubuntu/Debian: sudo apt-get install jq"
+  echo "  MacOS: brew install jq"
+  exit 1
+fi
+
+echo
+echo
+echo "===============  CLEAN TOKEN LIST ==============="
+echo
+echo
+
+# Get input path and chainId
+read -p "Enter path to the raw token list file >>> " INPUT_PATH
+read -p "Enter chainId to filter (leave blank for all chains) >>> " CHAIN_ID
+
+# Set output path using TEMPLATE_DIR
+OUTPUT_PATH="$TEMPLATE_DIR/lists/token_list.json"
+
+# Ask user to confirm and proceed
+echo
+echo "ℹ️ Confirm if this is correct:"
+echo
+printf "%30s %5s\n" "Clean token list FROM:" "$INPUT_PATH"
+printf "%30s %5s\n" "Filter for chainId:" "$CHAIN_ID"
+printf "%30s %5s\n" "Save cleaned list TO:" "${INPUT_PATH%.*}_CLEANED.json"
+echo
+
+prompt_proceed
+if [[ "$PROCEED" == "Y" || "$PROCEED" == "y" ]]
+then
+  clean_token_list "$INPUT_PATH" "$CHAIN_ID"
+else
+  echo "Exiting..."
+  exit
+fi