Merge pull request #904 from ScrapeGraphAI/pre/beta

Pre/beta

CHANGELOG.md

@@ -1,18 +1,29 @@
-## [1.37.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.36.0...v1.37.0) (2025-01-21)
+## [1.37.1-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.37.0...v1.37.1-beta.1) (2025-01-22)
 
 
-### Features
+### Bug Fixes
+
+* Schema parameter type ([2b5bd80](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2b5bd80a945a24072e578133eacc751feeec6188))
+
+
+### CI
+
+* **release:** 1.36.1-beta.1 [skip ci] ([006a2aa](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/006a2aaa3fbafbd5b2030c48d5b04b605532c06f))
+
+## [1.36.1-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.36.0...v1.36.1-beta.1) (2025-01-21)
 
-* add integration for search on web ([224ff07](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/224ff07032d006d75160a7094366fac17023aca1))
 
 
 ### Bug Fixes
 
+* Schema parameter type ([2b5bd80](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2b5bd80a945a24072e578133eacc751feeec6188))
 * search ([ce25b6a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/ce25b6a4b0e1ea15edf14a5867f6336bb27590cb))
 
 
+
 ### Docs
 
+
 * add requirements.dev ([6e12981](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/6e12981e637d078a6d3b3ce83f0d4901e9dd9996))
 * added first ollama example ([aa6a76e](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/aa6a76e5bdf63544f62786b0d17effa205aab3d8))
 

codebeaver.yml

@@ -0,0 +1,2 @@
+from: pytest
+setup_commands: ['@merge', 'pip install -q selenium', 'pip install -q playwright', 'playwright install']

pyproject.toml

@@ -1,6 +1,7 @@
 [project]
 name = "scrapegraphai"
-version = "1.37.0"
+
+version = "1.37.1b1"
 
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."

scrapegraphai/graphs/abstract_graph.py

@@ -6,7 +6,7 @@
 import uuid
 import warnings
 from abc import ABC, abstractmethod
-from typing import Optional
+from typing import Optional, Type
 
 from langchain.chat_models import init_chat_model
 from langchain_core.rate_limiters import InMemoryRateLimiter
@@ -51,7 +51,7 @@ def __init__(
         prompt: str,
         config: dict,
         source: Optional[str] = None,
-        schema: Optional[BaseModel] = None,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         if config.get("llm").get("temperature") is None:
             config["llm"]["temperature"] = 0

scrapegraphai/graphs/code_generator_graph.py

@@ -2,7 +2,7 @@
 SmartScraperGraph Module
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -56,7 +56,11 @@ class CodeGeneratorGraph(AbstractGraph):
     """
 
     def __init__(
-        self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None
+        self,
+        prompt: str,
+        source: str,
+        config: dict,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         super().__init__(prompt, config, source, schema)
 

scrapegraphai/graphs/csv_scraper_graph.py

@@ -2,7 +2,7 @@
 Module for creating the smart scraper
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -22,7 +22,7 @@ class CSVScraperGraph(AbstractGraph):
         config (dict): Additional configuration parameters needed by some nodes in the graph.
 
     Methods:
-        __init__ (prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None):
+        __init__ (prompt: str, source: str, config: dict, schema: Optional[Type[BaseModel]] = None):
             Initializes the CSVScraperGraph with a prompt, source, and configuration.
 
         __init__ initializes the CSVScraperGraph class. It requires the user's prompt as input,
@@ -49,7 +49,11 @@ class CSVScraperGraph(AbstractGraph):
     """
 
     def __init__(
-        self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None
+        self,
+        prompt: str,
+        source: str,
+        config: dict,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         """
         Initializes the CSVScraperGraph with a prompt, source, and configuration.

scrapegraphai/graphs/csv_scraper_multi_graph.py

@@ -3,7 +3,7 @@
 """
 
 from copy import deepcopy
-from typing import List, Optional
+from typing import List, Optional, Type
 
 from pydantic import BaseModel
 
@@ -47,7 +47,7 @@ def __init__(
         prompt: str,
         source: List[str],
         config: dict,
-        schema: Optional[BaseModel] = None,
+        schema: Optional[Type[BaseModel]] = None,
     ):
 
         self.copy_config = safe_deepcopy(config)

scrapegraphai/graphs/depth_search_graph.py

@@ -2,7 +2,7 @@
 depth search graph Module
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -54,7 +54,11 @@ class DepthSearchGraph(AbstractGraph):
     """
 
     def __init__(
-        self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None
+        self,
+        prompt: str,
+        source: str,
+        config: dict,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         super().__init__(prompt, config, source, schema)
 

scrapegraphai/graphs/document_scraper_graph.py

@@ -2,7 +2,7 @@
 This module implements the Document Scraper Graph for the ScrapeGraphAI application.
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -44,7 +44,11 @@ class DocumentScraperGraph(AbstractGraph):
     """
 
     def __init__(
-        self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None
+        self,
+        prompt: str,
+        source: str,
+        config: dict,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         super().__init__(prompt, config, source, schema)
 

scrapegraphai/graphs/document_scraper_multi_graph.py

@@ -3,7 +3,7 @@
 """
 
 from copy import deepcopy
-from typing import List, Optional
+from typing import List, Optional, Type
 
 from pydantic import BaseModel
 
@@ -47,7 +47,7 @@ def __init__(
         prompt: str,
         source: List[str],
         config: dict,
-        schema: Optional[BaseModel] = None,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         self.copy_config = safe_deepcopy(config)
         self.copy_schema = deepcopy(schema)

scrapegraphai/graphs/json_scraper_graph.py

@@ -2,7 +2,7 @@
 JSONScraperGraph Module
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -42,7 +42,11 @@ class JSONScraperGraph(AbstractGraph):
     """
 
     def __init__(
-        self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None
+        self,
+        prompt: str,
+        source: str,
+        config: dict,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         super().__init__(prompt, config, source, schema)
 

scrapegraphai/graphs/json_scraper_multi_graph.py

@@ -3,7 +3,7 @@
 """
 
 from copy import deepcopy
-from typing import List, Optional
+from typing import List, Optional, Type
 
 from pydantic import BaseModel
 
@@ -47,7 +47,7 @@ def __init__(
         prompt: str,
         source: List[str],
         config: dict,
-        schema: Optional[BaseModel] = None,
+        schema: Optional[Type[BaseModel]] = None,
     ):
 
         self.copy_config = safe_deepcopy(config)

scrapegraphai/graphs/omni_scraper_graph.py

@@ -2,7 +2,7 @@
 This module implements the Omni Scraper Graph for the ScrapeGraphAI application.
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -47,7 +47,11 @@ class OmniScraperGraph(AbstractGraph):
     """
 
     def __init__(
-        self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None
+        self,
+        prompt: str,
+        source: str,
+        config: dict,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         self.max_images = 5 if config is None else config.get("max_images", 5)
 

scrapegraphai/graphs/omni_search_graph.py

@@ -3,7 +3,7 @@
 """
 
 from copy import deepcopy
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -41,7 +41,9 @@ class OmniSearchGraph(AbstractGraph):
         >>> result = search_graph.run()
     """
 
-    def __init__(self, prompt: str, config: dict, schema: Optional[BaseModel] = None):
+    def __init__(
+        self, prompt: str, config: dict, schema: Optional[Type[BaseModel]] = None
+    ):
 
         self.max_results = config.get("max_results", 3)
 

scrapegraphai/graphs/screenshot_scraper_graph.py

@@ -2,7 +2,7 @@
 ScreenshotScraperGraph Module
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -21,7 +21,7 @@ class ScreenshotScraperGraph(AbstractGraph):
         source (str): The source URL or image link to scrape from.
 
     Methods:
-        __init__(prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None)
+        __init__(prompt: str, source: str, config: dict, schema: Optional[Type[BaseModel]] = None)
             Initializes the ScreenshotScraperGraph instance with the given prompt,
             source, and configuration parameters.
 
@@ -33,7 +33,11 @@ class ScreenshotScraperGraph(AbstractGraph):
     """
 
     def __init__(
-        self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None
+        self,
+        prompt: str,
+        source: str,
+        config: dict,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         super().__init__(prompt, config, source, schema)
 

scrapegraphai/graphs/script_creator_graph.py

@@ -2,7 +2,7 @@
 ScriptCreatorGraph Module
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -44,7 +44,11 @@ class ScriptCreatorGraph(AbstractGraph):
     """
 
     def __init__(
-        self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None
+        self,
+        prompt: str,
+        source: str,
+        config: dict,
+        schema: Optional[Type[BaseModel]] = None,
     ):
         self.library = config["library"]
 

scrapegraphai/graphs/script_creator_multi_graph.py

@@ -3,7 +3,7 @@
 """
 
 from copy import deepcopy
-from typing import List, Optional
+from typing import List, Optional, Type
 
 from pydantic import BaseModel
 
@@ -46,7 +46,7 @@ def __init__(
         prompt: str,
         source: List[str],
         config: dict,
-        schema: Optional[BaseModel] = None,
+        schema: Optional[Type[BaseModel]] = None,
     ):
 
         self.copy_config = safe_deepcopy(config)

scrapegraphai/graphs/search_graph.py

@@ -3,7 +3,7 @@
 """
 
 from copy import deepcopy
-from typing import List, Optional
+from typing import List, Optional, Type
 
 from pydantic import BaseModel
 
@@ -42,7 +42,9 @@ class SearchGraph(AbstractGraph):
         >>> print(search_graph.get_considered_urls())
     """
 
-    def __init__(self, prompt: str, config: dict, schema: Optional[BaseModel] = None):
+    def __init__(
+        self, prompt: str, config: dict, schema: Optional[Type[BaseModel]] = None
+    ):
         self.max_results = config.get("max_results", 3)
 
         self.copy_config = safe_deepcopy(config)

scrapegraphai/graphs/search_link_graph.py

@@ -2,7 +2,7 @@
 SearchLinkGraph Module
 """
 
-from typing import Optional
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -36,7 +36,9 @@ class SearchLinkGraph(AbstractGraph):
 
     """
 
-    def __init__(self, source: str, config: dict, schema: Optional[BaseModel] = None):
+    def __init__(
+        self, source: str, config: dict, schema: Optional[Type[BaseModel]] = None
+    ):
         super().__init__("", config, source, schema)
 
         self.input_key = "url" if source.startswith("http") else "local_dir"