From e3f17d68c75d75d79591d3e6e65d254e54b67f96 Mon Sep 17 00:00:00 2001 From: Salar Rahmanian Date: Sat, 20 May 2023 21:28:25 -0700 Subject: [PATCH 1/5] add client to fetch rss feeds from a given rss url --- build.sbt | 5 ++- .../com/softinio/scalanews/HttpClient.scala | 43 +++++++++++++++++++ .../scala/com/softinio/scalanews/Rome.scala | 38 ++++++++++++++++ .../softinio/scalanews/HttpClientSuite.scala | 35 +++++++++++++++ .../com/softinio/scalanews/RomeSuite.scala | 39 +++++++++++++++++ 5 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 core/src/main/scala/com/softinio/scalanews/HttpClient.scala create mode 100644 core/src/main/scala/com/softinio/scalanews/Rome.scala create mode 100644 core/src/test/scala/com/softinio/scalanews/HttpClientSuite.scala create mode 100644 core/src/test/scala/com/softinio/scalanews/RomeSuite.scala diff --git a/build.sbt b/build.sbt index 2d71965..10335f2 100644 --- a/build.sbt +++ b/build.sbt @@ -40,11 +40,14 @@ lazy val core = crossProject(JVMPlatform) name := "scalanews", libraryDependencies ++= Seq( "org.typelevel" %% "cats-core" % "2.9.0", - "org.typelevel" %% "cats-effect" % "3.4.10", + "org.typelevel" %% "cats-effect" % "3.5.0", "io.github.akiomik" %% "cats-nio-file" % "1.7.0", "com.monovore" %% "decline-effect" % "2.4.1", "com.github.pureconfig" %% "pureconfig" % "0.17.3", "com.github.pureconfig" %% "pureconfig-cats-effect" % "0.17.3", + "org.http4s" %% "http4s-ember-client" % "0.23.19", + "org.http4s" %% "http4s-dsl" % "0.23.19", + "com.rometools" % "rome" % "2.1.0", "org.scalameta" %% "munit" % "0.7.29" % Test, "org.typelevel" %% "munit-cats-effect-3" % "1.0.7" % Test ), diff --git a/core/src/main/scala/com/softinio/scalanews/HttpClient.scala b/core/src/main/scala/com/softinio/scalanews/HttpClient.scala new file mode 100644 index 0000000..4cdab04 --- /dev/null +++ b/core/src/main/scala/com/softinio/scalanews/HttpClient.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Salar Rahmanian + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.softinio.scalanews + +import cats.effect._ +import org.http4s.client.Client +import org.http4s.ember.client.EmberClientBuilder +import java.io.InputStream +import org.http4s.Request +import org.http4s.Method +import org.http4s.Uri +import fs2._ + +object HttpClient { + + def fetchRss(feedUrl: String): Resource[IO, InputStream] = { + val request = Request[IO](Method.GET, Uri.unsafeFromString(feedUrl)) + client.flatMap { client => + client + .run(request) + .flatMap(response => io.toInputStreamResource(response.body)) + } + } + + private lazy val client: Resource[IO, Client[IO]] = + EmberClientBuilder + .default[IO] + .build +} diff --git a/core/src/main/scala/com/softinio/scalanews/Rome.scala b/core/src/main/scala/com/softinio/scalanews/Rome.scala new file mode 100644 index 0000000..b20b087 --- /dev/null +++ b/core/src/main/scala/com/softinio/scalanews/Rome.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2023 Salar Rahmanian + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.softinio.scalanews + +import java.io.InputStream + +import cats.effect._ + +import com.rometools.rome.feed.synd.SyndFeed +import com.rometools.rome.io.SyndFeedInput +import com.rometools.rome.io.XmlReader + +object Rome { + final private def parseFeed( + feedStream: InputStream + ): IO[Either[Throwable, SyndFeed]] = + IO.blocking { + val input: SyndFeedInput = new SyndFeedInput() + input.build(new XmlReader(feedStream)) + }.attempt + + def fetchFeed(feedUrl: String): IO[Either[Throwable, SyndFeed]] = + HttpClient.fetchRss(feedUrl).use(parseFeed(_)) +} diff --git a/core/src/test/scala/com/softinio/scalanews/HttpClientSuite.scala b/core/src/test/scala/com/softinio/scalanews/HttpClientSuite.scala new file mode 100644 index 0000000..0dc10f3 --- /dev/null +++ b/core/src/test/scala/com/softinio/scalanews/HttpClientSuite.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2023 Salar Rahmanian + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.softinio.scalanews + +import cats.effect._ +import munit.CatsEffectSuite + +import cats.effect.unsafe.IORuntime + +class HttpClientSuite extends CatsEffectSuite { + + implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global + test("Fetch Rss") { + val result = HttpClient.fetchRss("https://www.softinio.com/index.xml") + val obtained = result.use { res => + val resultStr = new String(res.readAllBytes) + IO(resultStr.contains("lightening-talks-at-pybay-2018")) + } + assertIO(obtained, true) + } +} diff --git a/core/src/test/scala/com/softinio/scalanews/RomeSuite.scala b/core/src/test/scala/com/softinio/scalanews/RomeSuite.scala new file mode 100644 index 0000000..2a25cd7 --- /dev/null +++ b/core/src/test/scala/com/softinio/scalanews/RomeSuite.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2023 Salar Rahmanian + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.softinio.scalanews + +import cats.effect._ + +import munit.CatsEffectSuite + +class RomeSuite extends CatsEffectSuite { + + test("Fetch Feed") { + val obtained: IO[Boolean] = for { + result <- Rome.fetchFeed("https://www.softinio.com/index.xml") + } yield { + result match { + case Right(feed) => { + val title = feed.getTitle + title == "Salar Rahmanian" + } + case _ => false + } + } + assertIO(obtained, true) + } +} From e5ef479299fc08d9112112ef84fbcfe55fe7b548 Mon Sep 17 00:00:00 2001 From: Salar Rahmanian Date: Sat, 9 Dec 2023 15:21:38 -0800 Subject: [PATCH 2/5] Package and tools updates --- build.sbt | 16 ++++---- flake.lock | 82 ++++++++++++++++++++++++---------------- project/build.properties | 2 +- 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/build.sbt b/build.sbt index 10335f2..2b14b76 100644 --- a/build.sbt +++ b/build.sbt @@ -25,7 +25,7 @@ ThisBuild / tlSitePublishBranch := Some("main") ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("17")) -val Scala213 = "2.13.8" +val Scala213 = "2.13.12" ThisBuild / crossScalaVersions := Seq(Scala213) ThisBuild / scalaVersion := Scala213 // the default Scala @@ -39,14 +39,14 @@ lazy val core = crossProject(JVMPlatform) .settings( name := "scalanews", libraryDependencies ++= Seq( - "org.typelevel" %% "cats-core" % "2.9.0", - "org.typelevel" %% "cats-effect" % "3.5.0", - "io.github.akiomik" %% "cats-nio-file" % "1.7.0", + "org.typelevel" %% "cats-core" % "2.10.0", + "org.typelevel" %% "cats-effect" % "3.5.2", + "io.github.akiomik" %% "cats-nio-file" % "1.10.0", "com.monovore" %% "decline-effect" % "2.4.1", - "com.github.pureconfig" %% "pureconfig" % "0.17.3", - "com.github.pureconfig" %% "pureconfig-cats-effect" % "0.17.3", - "org.http4s" %% "http4s-ember-client" % "0.23.19", - "org.http4s" %% "http4s-dsl" % "0.23.19", + "com.github.pureconfig" %% "pureconfig" % "0.17.4", + "com.github.pureconfig" %% "pureconfig-cats-effect" % "0.17.4", + "org.http4s" %% "http4s-ember-client" % "0.23.24", + "org.http4s" %% "http4s-dsl" % "0.23.24", "com.rometools" % "rome" % "2.1.0", "org.scalameta" %% "munit" % "0.7.29" % Test, "org.typelevel" %% "munit-cats-effect-3" % "1.0.7" % Test diff --git a/flake.lock b/flake.lock index 9d32ed5..af3f061 100644 --- a/flake.lock +++ b/flake.lock @@ -2,15 +2,15 @@ "nodes": { "devshell": { "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "systems": "systems" }, "locked": { - "lastModified": 1666548262, - "narHash": "sha256-4DyN4KXqQQsCw0vCXkMThw4b5Q4/q87ZZgRb4st8COc=", + "lastModified": 1700815693, + "narHash": "sha256-JtKZEQUzosrCwDsLgm+g6aqbP1aseUl1334OShEAS3s=", "owner": "numtide", "repo": "devshell", - "rev": "c8ce8ed81726079c398f5f29c4b68a7d6a3c2fa2", + "rev": "7ad1c417c87e98e56dcef7ecd0e0a2f2e5669d51", "type": "github" }, "original": { @@ -20,27 +20,15 @@ } }, "flake-utils": { - "locked": { - "lastModified": 1642700792, - "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba", - "type": "github" + "inputs": { + "systems": "systems_2" }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_2": { "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", "owner": "numtide", "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", "type": "github" }, "original": { @@ -51,11 +39,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1643381941, - "narHash": "sha256-pHTwvnN4tTsEKkWlXQ8JMY423epos8wUOhthpwJjtpc=", + "lastModified": 1677383253, + "narHash": "sha256-UfpzWfSxkfXHnb4boXZNaKsAcUrZT9Hw+tao1oZxd08=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5efc8ca954272c4376ac929f4c5ffefcc20551d5", + "rev": "9952d6bc395f5841262b006fbace8dd7e143b634", "type": "github" }, "original": { @@ -67,11 +55,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1666570118, - "narHash": "sha256-MTXmIYowHM1wyIYyqPdBLia5SjGnxETv0YkIbDsbkx4=", + "lastModified": 1701626906, + "narHash": "sha256-ugr1QyzzwNk505ICE4VMQzonHQ9QS5W33xF2FXzFQ00=", "owner": "nixos", "repo": "nixpkgs", - "rev": "1e684b371cf05300bc2b432f958f285855bac8fb", + "rev": "0c6d8c783336a59f4c59d4a6daed6ab269c4b361", "type": "github" }, "original": { @@ -94,18 +82,48 @@ "typelevel-nix": "typelevel-nix" } }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, "typelevel-nix": { "inputs": { "devshell": "devshell", - "flake-utils": "flake-utils_2", + "flake-utils": "flake-utils", "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1666614477, - "narHash": "sha256-Qk+UgRgj4oKiNfvoxeAub/5KMfBTVvaVRmTIAMcpGtw=", + "lastModified": 1701880040, + "narHash": "sha256-7yKDDltsZHdF8xKYHKUGl6C3Sfcsb7NDOTAOAxkjgTo=", "owner": "typelevel", "repo": "typelevel-nix", - "rev": "1be908b370f7fc01f2f05528eecb0536a3b12441", + "rev": "1cf38c7377dbf9e2cb0d20bbbe9e4d994a228343", "type": "github" }, "original": { diff --git a/project/build.properties b/project/build.properties index 8b9a0b0..e8a1e24 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.8.0 +sbt.version=1.9.7 From 6d428f9be741045c4dda88ca41d2f0c06e2f8a72 Mon Sep 17 00:00:00 2001 From: Salar Rahmanian Date: Sat, 9 Dec 2023 15:23:20 -0800 Subject: [PATCH 3/5] Add ability to fetch blogs from rss urls and create a list by date range --- .../com/softinio/scalanews/Bloggers.scala | 71 +++++++++++++++++-- .../softinio/scalanews/algebra/Article.scala | 22 ++++++ .../softinio/scalanews/BloggersSuite.scala | 46 +++++++++++- 3 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 core/src/main/scala/com/softinio/scalanews/algebra/Article.scala diff --git a/core/src/main/scala/com/softinio/scalanews/Bloggers.scala b/core/src/main/scala/com/softinio/scalanews/Bloggers.scala index 4340d27..43a5624 100644 --- a/core/src/main/scala/com/softinio/scalanews/Bloggers.scala +++ b/core/src/main/scala/com/softinio/scalanews/Bloggers.scala @@ -18,13 +18,19 @@ package com.softinio.scalanews import java.nio.file.Paths import java.nio.file.StandardOpenOption +import java.util.Date import cats.effect._ import cats.nio.file.Files +import com.rometools.rome.feed.synd.SyndEntry +import org.http4s.Uri +import scala.jdk.CollectionConverters._ +import com.softinio.scalanews.algebra.Article import com.softinio.scalanews.algebra.Blog object Bloggers { - val directoryMarkdownFilePath = Paths.get("docs/Resources/Blog_Directory.md") + private val directoryMarkdownFilePath = + Paths.get("docs/Resources/Blog_Directory.md") def generateDirectory(bloggerList: List[Blog]): IO[String] = { IO.blocking { val header = """ @@ -45,12 +51,69 @@ object Bloggers { } s""" - ${header} + $header ${directory.mkString("\n")} - ${footer}\n""".stripMargin + $footer\n""".stripMargin } } + private def getArticlesFromEntries( + entries: List[SyndEntry], + startDate: Date, + endDate: Date + ): Option[List[Article]] = + entries + .filter(_.getPublishedDate != null) + .filter(_.getLink != null) + .filter(_.getTitle != null) + .map { entry => + Article( + entry.getTitle, + Uri + .fromString(entry.getLink) + .getOrElse(Uri.unsafeFromString("https://www.scala-lang.org/")), + entry.getPublishedDate + ) + } + .filter { case Article(_, _, publishedDate) => + publishedDate.after(startDate) && publishedDate.before(endDate) + } + .distinct + .sortBy(_.publishedDate.getTime) + .reverse match { + case Nil => None + case list => Some(list) + } + + def getArticlesForBlogger( + blog: Blog, + startDate: Date, + endDate: Date + ): IO[Option[List[Article]]] = + for { + feedResult <- Rome.fetchFeed(blog.rss.toURL.toString) + } yield { + getArticlesFromEntries( + feedResult + .map(_.getEntries.asScala.toList) + .getOrElse(List[SyndEntry]()), + startDate, + endDate + ) + } + + def createBlogList(startDate: Date, endDate: Date): IO[List[Article]] = + ConfigLoader.load().flatMap { conf => + conf.bloggers.foldLeft(IO.pure(List[Article]()))((acc, blog) => + acc.flatMap { articleList => + getArticlesForBlogger(blog, startDate, endDate).map { + maybeArticleList => + articleList ++ maybeArticleList.getOrElse(List[Article]()) + } + } + ) + } + def createBloggerDirectory(bloggerList: List[Blog]): IO[ExitCode] = { for { exists <- Files[IO].exists(directoryMarkdownFilePath) @@ -61,6 +124,6 @@ object Bloggers { directory.getBytes(), StandardOpenOption.CREATE_NEW ) - } yield (ExitCode.Success) + } yield ExitCode.Success } } diff --git a/core/src/main/scala/com/softinio/scalanews/algebra/Article.scala b/core/src/main/scala/com/softinio/scalanews/algebra/Article.scala new file mode 100644 index 0000000..9f78838 --- /dev/null +++ b/core/src/main/scala/com/softinio/scalanews/algebra/Article.scala @@ -0,0 +1,22 @@ +/* + * Copyright 2023 Salar Rahmanian + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.softinio.scalanews.algebra + +import java.util.Date +import org.http4s.Uri + +case class Article(title: String, url: Uri, publishedDate: Date) diff --git a/core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala b/core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala index ebe79a8..735d80b 100644 --- a/core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala +++ b/core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala @@ -16,11 +16,11 @@ package com.softinio.scalanews -import java.net.URI - +import com.softinio.scalanews.algebra.Blog import munit.CatsEffectSuite -import com.softinio.scalanews.algebra.Blog +import java.net.URI +import java.text.SimpleDateFormat class BloggersSuite extends CatsEffectSuite { test("generateDirectory - test the new blogger directory is generated") { @@ -36,4 +36,44 @@ class BloggersSuite extends CatsEffectSuite { )) assertIO(obtained, true) } + + test( + "getArticlesForBlogger - test getting articles for a blog list for a blogger for a given date range" + ) { + val formatter = new SimpleDateFormat("yyyy-MM-dd") + val blog = Blog( + "Salar Rahmanian", + new URI("https://www.softinio.com"), + new URI("https://www.softinio.com/index.xml") + ) + val obtained = for { + result <- Bloggers.getArticlesForBlogger( + blog, + formatter.parse("2021-01-01"), + formatter.parse("2021-12-31") + ) + } yield { + result match { + case Some(articles) => articles.nonEmpty + case None => false + } + } + assertIO(obtained, true) + } + + test( + "createBlogList - test getting articles for a blog list for all bloggers for a given date range" + ) { + val formatter = new SimpleDateFormat("yyyy-MM-dd") + + val obtained = for { + result <- Bloggers.createBlogList( + formatter.parse("2021-01-01"), + formatter.parse("2021-12-31") + ) + } yield { + result.nonEmpty + } + assertIO(obtained, true) + } } From c6eed18addec339d6b92bbdae5bb7d282c0dc269 Mon Sep 17 00:00:00 2001 From: Salar Rahmanian Date: Sat, 9 Dec 2023 15:25:35 -0800 Subject: [PATCH 4/5] update ci with newer scala version --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c9a768..3b86b11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - scala: [2.13.8] + scala: [2.13.12] java: [temurin@17] project: [rootJVM] runs-on: ${{ matrix.os }} @@ -89,7 +89,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - scala: [2.13.8] + scala: [2.13.12] java: [temurin@17] runs-on: ${{ matrix.os }} steps: From 4cb5644b1e46a9f64710257490eaa33a6975b3bd Mon Sep 17 00:00:00 2001 From: Salar Rahmanian Date: Sat, 9 Dec 2023 15:25:58 -0800 Subject: [PATCH 5/5] Linting --- .../src/test/scala/com/softinio/scalanews/BloggersSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala b/core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala index 735d80b..501cc23 100644 --- a/core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala +++ b/core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala @@ -31,9 +31,9 @@ class BloggersSuite extends CatsEffectSuite { ) val obtained = for { result <- Bloggers.generateDirectory(List(blog)) - } yield (result.contains( + } yield result.contains( "| Salar Rahmanian | | [rss feed](https://www.softinio.com/index.xml) |" - )) + ) assertIO(obtained, true) }