Refactor string sanitization logic to remove invisible characters from news data.

apis/newsapi/main.go

@@ -7,21 +7,11 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"unicode"
 )
 
 const apiBaseURL = "https://newsapi.org/v2/top-headlines"
 
-var forbiddenStrings = []string{
-	"\r", "\\r", "\ufeff", "\u00A0", "\u200b", "\u200c",
-	"\u200d", // Zero Width Joiner
-	"\u200e", // Left-to-Right Mark
-	"\u200f", // Right-to-Left Mark
-	"\u2060", // Word Joiner
-	"\ufe0f", // Variation Selector-16
-	"\u2028",
-	"\u2029",
-}
-
 type configFile struct {
 	News newsConfig
 }
@@ -91,19 +81,45 @@ func FetchNews() News {
 
 	news := News{}
 
-	err = json.Unmarshal([]byte(removeForbiddenStrings(string(respBody))), &news)
+	err = json.Unmarshal(respBody, &news)
-
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	sanitizeNews(&news)
+
 	return news
 }
 
-func removeForbiddenStrings(s string) string {
+func sanitizeNews(news *News) {
-	result := s
+	news.Status = removeInvisibleCharacters(news.Status)
-	for _, forbidden := range forbiddenStrings {
+
-		result = strings.ReplaceAll(result, forbidden, "")
+	for i := range news.Articles {
+		news.Articles[i].Author = removeInvisibleCharacters(news.Articles[i].Author)
+		news.Articles[i].Title = removeInvisibleCharacters(news.Articles[i].Title)
+		news.Articles[i].Description = removeInvisibleCharacters(news.Articles[i].Description)
+		news.Articles[i].URL = removeInvisibleCharacters(news.Articles[i].URL)
+		news.Articles[i].URLToImage = removeInvisibleCharacters(news.Articles[i].URLToImage)
+		news.Articles[i].PublishedAt = removeInvisibleCharacters(news.Articles[i].PublishedAt)
+		news.Articles[i].Content = removeInvisibleCharacters(news.Articles[i].Content)
+		news.Articles[i].Source.ID = removeInvisibleCharacters(news.Articles[i].Source.ID)
+		news.Articles[i].Source.Name = removeInvisibleCharacters(news.Articles[i].Source.Name)
 	}
-	return result
+}
+
+func removeInvisibleCharacters(s string) string {
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case '\u00A0':
+			return ' '
+		case '\r', '\u2028', '\u2029':
+			return ' '
+		}
+
+		if unicode.Is(unicode.Cf, r) {
+			return ' '
+		}
+
+		return r
+	}, s)
 }