Merge pull request #30 from softinio/add-rss-fetch

Add ability to fetch blogs using rss links

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        scala: [2.13.8]
+        scala: [2.13.12]
         java: [temurin@17]
         project: [rootJVM]
     runs-on: ${{ matrix.os }}
@@ -89,7 +89,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        scala: [2.13.8]
+        scala: [2.13.12]
         java: [temurin@17]
     runs-on: ${{ matrix.os }}
     steps:

build.sbt

@@ -25,7 +25,7 @@ ThisBuild / tlSitePublishBranch := Some("main")
 
 ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("17"))
 
-val Scala213 = "2.13.8"
+val Scala213 = "2.13.12"
 ThisBuild / crossScalaVersions := Seq(Scala213)
 ThisBuild / scalaVersion := Scala213 // the default Scala
 
@@ -39,12 +39,15 @@ lazy val core = crossProject(JVMPlatform)
   .settings(
     name := "scalanews",
     libraryDependencies ++= Seq(
-      "org.typelevel" %% "cats-core" % "2.9.0",
-      "org.typelevel" %% "cats-effect" % "3.4.10",
-      "io.github.akiomik" %% "cats-nio-file" % "1.7.0",
+      "org.typelevel" %% "cats-core" % "2.10.0",
+      "org.typelevel" %% "cats-effect" % "3.5.2",
+      "io.github.akiomik" %% "cats-nio-file" % "1.10.0",
       "com.monovore" %% "decline-effect" % "2.4.1",
-      "com.github.pureconfig" %% "pureconfig" % "0.17.3",
-      "com.github.pureconfig" %% "pureconfig-cats-effect" % "0.17.3",
+      "com.github.pureconfig" %% "pureconfig" % "0.17.4",
+      "com.github.pureconfig" %% "pureconfig-cats-effect" % "0.17.4",
+      "org.http4s" %% "http4s-ember-client" % "0.23.24",
+      "org.http4s" %% "http4s-dsl" % "0.23.24",
+      "com.rometools" % "rome" % "2.1.0",
       "org.scalameta" %% "munit" % "0.7.29" % Test,
       "org.typelevel" %% "munit-cats-effect-3" % "1.0.7" % Test
     ),

core/src/main/scala/com/softinio/scalanews/Bloggers.scala

@@ -18,13 +18,19 @@ package com.softinio.scalanews
 
 import java.nio.file.Paths
 import java.nio.file.StandardOpenOption
+import java.util.Date
 import cats.effect._
 import cats.nio.file.Files
+import com.rometools.rome.feed.synd.SyndEntry
+import org.http4s.Uri
 
+import scala.jdk.CollectionConverters._
+import com.softinio.scalanews.algebra.Article
 import com.softinio.scalanews.algebra.Blog
 
 object Bloggers {
-  val directoryMarkdownFilePath = Paths.get("docs/Resources/Blog_Directory.md")
+  private val directoryMarkdownFilePath =
+    Paths.get("docs/Resources/Blog_Directory.md")
   def generateDirectory(bloggerList: List[Blog]): IO[String] = {
     IO.blocking {
       val header = """
@@ -45,12 +51,69 @@ object Bloggers {
       }
 
       s"""
-      ${header}
+      $header
       ${directory.mkString("\n")}
-      ${footer}\n""".stripMargin
+      $footer\n""".stripMargin
     }
   }
 
+  private def getArticlesFromEntries(
+      entries: List[SyndEntry],
+      startDate: Date,
+      endDate: Date
+  ): Option[List[Article]] =
+    entries
+      .filter(_.getPublishedDate != null)
+      .filter(_.getLink != null)
+      .filter(_.getTitle != null)
+      .map { entry =>
+        Article(
+          entry.getTitle,
+          Uri
+            .fromString(entry.getLink)
+            .getOrElse(Uri.unsafeFromString("https://www.scala-lang.org/")),
+          entry.getPublishedDate
+        )
+      }
+      .filter { case Article(_, _, publishedDate) =>
+        publishedDate.after(startDate) && publishedDate.before(endDate)
+      }
+      .distinct
+      .sortBy(_.publishedDate.getTime)
+      .reverse match {
+      case Nil  => None
+      case list => Some(list)
+    }
+
+  def getArticlesForBlogger(
+      blog: Blog,
+      startDate: Date,
+      endDate: Date
+  ): IO[Option[List[Article]]] =
+    for {
+      feedResult <- Rome.fetchFeed(blog.rss.toURL.toString)
+    } yield {
+      getArticlesFromEntries(
+        feedResult
+          .map(_.getEntries.asScala.toList)
+          .getOrElse(List[SyndEntry]()),
+        startDate,
+        endDate
+      )
+    }
+
+  def createBlogList(startDate: Date, endDate: Date): IO[List[Article]] =
+    ConfigLoader.load().flatMap { conf =>
+      conf.bloggers.foldLeft(IO.pure(List[Article]()))((acc, blog) =>
+        acc.flatMap { articleList =>
+          getArticlesForBlogger(blog, startDate, endDate).map {
+            maybeArticleList =>
+              articleList ++ maybeArticleList.getOrElse(List[Article]())
+          }
+        }
+      )
+    }
+
   def createBloggerDirectory(bloggerList: List[Blog]): IO[ExitCode] = {
     for {
       exists <- Files[IO].exists(directoryMarkdownFilePath)
@@ -61,6 +124,6 @@ object Bloggers {
         directory.getBytes(),
         StandardOpenOption.CREATE_NEW
       )
-    } yield (ExitCode.Success)
+    } yield ExitCode.Success
   }
 }

core/src/main/scala/com/softinio/scalanews/HttpClient.scala

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 Salar Rahmanian
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.softinio.scalanews
+
+import cats.effect._
+import org.http4s.client.Client
+import org.http4s.ember.client.EmberClientBuilder
+import java.io.InputStream
+import org.http4s.Request
+import org.http4s.Method
+import org.http4s.Uri
+import fs2._
+
+object HttpClient {
+
+  def fetchRss(feedUrl: String): Resource[IO, InputStream] = {
+    val request = Request[IO](Method.GET, Uri.unsafeFromString(feedUrl))
+    client.flatMap { client =>
+      client
+        .run(request)
+        .flatMap(response => io.toInputStreamResource(response.body))
+    }
+  }
+
+  private lazy val client: Resource[IO, Client[IO]] =
+    EmberClientBuilder
+      .default[IO]
+      .build
+}

core/src/main/scala/com/softinio/scalanews/Rome.scala

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 Salar Rahmanian
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.softinio.scalanews
+
+import java.io.InputStream
+
+import cats.effect._
+
+import com.rometools.rome.feed.synd.SyndFeed
+import com.rometools.rome.io.SyndFeedInput
+import com.rometools.rome.io.XmlReader
+
+object Rome {
+  final private def parseFeed(
+      feedStream: InputStream
+  ): IO[Either[Throwable, SyndFeed]] =
+    IO.blocking {
+      val input: SyndFeedInput = new SyndFeedInput()
+      input.build(new XmlReader(feedStream))
+    }.attempt
+
+  def fetchFeed(feedUrl: String): IO[Either[Throwable, SyndFeed]] =
+    HttpClient.fetchRss(feedUrl).use(parseFeed(_))
+}

core/src/main/scala/com/softinio/scalanews/algebra/Article.scala

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 Salar Rahmanian
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.softinio.scalanews.algebra
+
+import java.util.Date
+import org.http4s.Uri
+
+case class Article(title: String, url: Uri, publishedDate: Date)

core/src/test/scala/com/softinio/scalanews/BloggersSuite.scala

@@ -16,11 +16,11 @@
 
 package com.softinio.scalanews
 
-import java.net.URI
-
+import com.softinio.scalanews.algebra.Blog
 import munit.CatsEffectSuite
 
-import com.softinio.scalanews.algebra.Blog
+import java.net.URI
+import java.text.SimpleDateFormat
 
 class BloggersSuite extends CatsEffectSuite {
   test("generateDirectory - test the new blogger directory is generated") {
@@ -31,9 +31,49 @@ class BloggersSuite extends CatsEffectSuite {
     )
     val obtained = for {
       result <- Bloggers.generateDirectory(List(blog))
-    } yield (result.contains(
+    } yield result.contains(
       "| Salar Rahmanian | <https://www.softinio.com> | [rss feed](https://www.softinio.com/index.xml) |"
-    ))
+    )
+    assertIO(obtained, true)
+  }
+
+  test(
+    "getArticlesForBlogger - test getting articles for a blog list for a blogger for a given date range"
+  ) {
+    val formatter = new SimpleDateFormat("yyyy-MM-dd")
+    val blog = Blog(
+      "Salar Rahmanian",
+      new URI("https://www.softinio.com"),
+      new URI("https://www.softinio.com/index.xml")
+    )
+    val obtained = for {
+      result <- Bloggers.getArticlesForBlogger(
+        blog,
+        formatter.parse("2021-01-01"),
+        formatter.parse("2021-12-31")
+      )
+    } yield {
+      result match {
+        case Some(articles) => articles.nonEmpty
+        case None           => false
+      }
+    }
+    assertIO(obtained, true)
+  }
+
+  test(
+    "createBlogList - test getting articles for a blog list for all bloggers for a given date range"
+  ) {
+    val formatter = new SimpleDateFormat("yyyy-MM-dd")
+
+    val obtained = for {
+      result <- Bloggers.createBlogList(
+        formatter.parse("2021-01-01"),
+        formatter.parse("2021-12-31")
+      )
+    } yield {
+      result.nonEmpty
+    }
     assertIO(obtained, true)
   }
 }

core/src/test/scala/com/softinio/scalanews/HttpClientSuite.scala

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2023 Salar Rahmanian
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.softinio.scalanews
+
+import cats.effect._
+import munit.CatsEffectSuite
+
+import cats.effect.unsafe.IORuntime
+
+class HttpClientSuite extends CatsEffectSuite {
+
+  implicit val runtime: IORuntime = cats.effect.unsafe.IORuntime.global
+  test("Fetch Rss") {
+    val result = HttpClient.fetchRss("https://www.softinio.com/index.xml")
+    val obtained = result.use { res =>
+      val resultStr = new String(res.readAllBytes)
+      IO(resultStr.contains("lightening-talks-at-pybay-2018"))
+    }
+    assertIO(obtained, true)
+  }
+}

core/src/test/scala/com/softinio/scalanews/RomeSuite.scala

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2023 Salar Rahmanian
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.softinio.scalanews
+
+import cats.effect._
+
+import munit.CatsEffectSuite
+
+class RomeSuite extends CatsEffectSuite {
+
+  test("Fetch Feed") {
+    val obtained: IO[Boolean] = for {
+      result <- Rome.fetchFeed("https://www.softinio.com/index.xml")
+    } yield {
+      result match {
+        case Right(feed) => {
+          val title = feed.getTitle
+          title == "Salar Rahmanian"
+        }
+        case _ => false
+      }
+    }
+    assertIO(obtained, true)
+  }
+}