From 02be603dccee500745e31cb566e864209c1bebdb Mon Sep 17 00:00:00 2001 From: orangix Date: Sun, 25 Jan 2026 05:08:10 +0100 Subject: [PATCH] refactor api package comments * use encoding/json for comment parsing * refactor by moving loop code to an UnmarshalJSON * use a preallocated array and indices to maintain order while using goroutines again, this was removed a while ago * use new struct in comment.hbs and contextComment.hbs * rewriteUrl partial to reduce rimgo-specific code in api * move RenderError into pages package to avoid import cycle between render and utils --- api/comments.go | 186 ++++++++++++++++-------------- api/user.go | 23 ++-- main.go | 8 +- pages/embed.go | 4 +- {utils => pages}/error.go | 5 +- pages/media.go | 4 +- pages/post.go | 4 +- pages/rss.go | 2 +- pages/tag.go | 4 +- pages/user.go | 18 +-- render/helpers.go | 29 ++++- utils/request.go | 37 ++++++ utils/rewriteUrl.go | 25 ++++ views/partials/comment.hbs | 16 +-- views/partials/contextComment.hbs | 6 +- 15 files changed, 234 insertions(+), 137 deletions(-) rename {utils => pages}/error.go (87%) create mode 100644 utils/rewriteUrl.go diff --git a/api/comments.go b/api/comments.go index 3243b94..931e7e0 100644 --- a/api/comments.go +++ b/api/comments.go @@ -1,13 +1,15 @@ package api import ( + "encoding/json" + "errors" + "fmt" "regexp" "strings" "sync" "time" "codeberg.org/rimgo/rimgo/utils" - "github.com/dustin/go-humanize" "github.com/microcosm-cc/bluemonday" "github.com/patrickmn/go-cache" "github.com/tidwall/gjson" @@ -15,49 +17,51 @@ import ( ) type Comment struct { - Comments []Comment - User User - Post Submission - Id string - Comment string - Upvotes int64 - Downvotes int64 - Platform string - CreatedAt string - RelTime string - UpdatedAt string - DeletedAt string + ID int `json:"id"` + ParentID int `json:"parent_id"` + Comment string `json:"comment"` + PostID string `json:"post_id"` + UpvoteCount int `json:"upvote_count"` + DownvoteCount int `json:"downvote_count"` + PointCount int `json:"point_count"` + Vote struct{} `json:"vote"` // ??? + PlatformID int `json:"platform_id"` + Platform string `json:"platform"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt time.Time `json:"deleted_at"` + Next struct{} `json:"next"` // ??? + Comments commentArray `json:"comments"` + AccountID int `json:"account_id"` + Account _ApiUser `json:"account"` + Post commentPost `json:"post"` +} + +type _ApiUser struct { + ID int `json:"id"` + Username string `json:"username"` + Avatar string `json:"avatar"` } func (client *Client) FetchComments(galleryID string) ([]Comment, error) { cacheData, found := client.Cache.Get(galleryID + "-comments") if found { - return cacheData.([]Comment), nil + return cacheData.(commentArray), nil } - data, err := utils.GetJSON("https://api.imgur.com/comment/v1/comments?client_id=" + client.ClientID + "&filter[post]=eq:" + galleryID + "&include=account,adconfig&per_page=30&sort=best") + data, err := utils.GetJSONNew("https://api.imgur.com/comment/v1/comments?client_id=" + client.ClientID + "&filter[post]=eq:" + galleryID + "&include=account,adconfig&per_page=30&sort=best") if err != nil { return []Comment{}, nil } - wg := sync.WaitGroup{} - comments := make([]Comment, 0) - data.Get("data").ForEach( - func(key, value gjson.Result) bool { - wg.Add(1) + var parsed commentApiResponse + err = json.Unmarshal(data, &parsed) + if err != nil { + return []Comment{}, err + } - go func() { - defer wg.Done() - comments = append(comments, parseComment(value)) - }() - - return true - }, - ) - wg.Wait() - - client.Cache.Set(galleryID+"-comments", comments, cache.DefaultExpiration) - return comments, nil + client.Cache.Set(galleryID+"-comments", parsed.Data, cache.DefaultExpiration) + return parsed.Data, nil } var imgurRe = regexp.MustCompile(`https?://imgur\.com/(gallery|a)?/(.*)`) @@ -67,51 +71,32 @@ var vidRe = regexp.MustCompile(`https?://i\.imgur\.com/(.*)\.(mp4|webm)`) var vidFormatRe = regexp.MustCompile(`\.(mp4|webm)`) var iImgurRe = regexp.MustCompile(`https?://i\.imgur\.com`) -func parseComment(data gjson.Result) Comment { - createdTime, _ := time.Parse("2006-01-02T15:04:05Z", data.Get("created_at").String()) - createdAt := createdTime.Format("January 2, 2006 3:04 PM") - updatedAt, _ := utils.FormatDate(data.Get("updated_at").String()) - deletedAt, _ := utils.FormatDate(data.Get("deleted_at").String()) +func parseComment(data json.RawMessage, out *Comment) { + err := json.Unmarshal(data, &out) + if err != nil { + panic(err) + } - userAvatar := strings.ReplaceAll(data.Get("account.avatar").String(), "https://i.imgur.com", "") + comment := &out.Comment + *comment = strings.ReplaceAll(*comment, "\n", "
") - wg := sync.WaitGroup{} - comments := make([]Comment, 0) - data.Get("comments").ForEach( - func(key, value gjson.Result) bool { - wg.Add(1) - - go func() { - defer wg.Done() - comments = append(comments, parseComment(value)) - }() - - return true - }, - ) - wg.Wait() - - comment := data.Get("comment").String() - - comment = strings.ReplaceAll(comment, "\n", "
") - - for _, match := range imgRe.FindAllString(comment, -1) { + for _, match := range imgRe.FindAllString(*comment, -1) { img := iImgurRe.ReplaceAllString(match, "") img = `` - comment = strings.Replace(comment, match, img, 1) + *comment = strings.Replace(*comment, match, img, 1) } - for _, match := range vidRe.FindAllString(comment, -1) { + for _, match := range vidRe.FindAllString(*comment, -1) { vid := iImgurRe.ReplaceAllString(match, "") vid = `` - comment = strings.Replace(comment, match, vid, 1) + *comment = strings.Replace(*comment, match, vid, 1) } - for _, l := range linkify.Links(comment) { - origLink := comment[l.Start:l.End] + for _, l := range linkify.Links(*comment) { + origLink := (*comment)[l.Start:l.End] link := `` + origLink + `` - comment = strings.Replace(comment, origLink, link, 1) + *comment = strings.Replace(*comment, origLink, link, 1) } - comment = imgurRe.ReplaceAllString(comment, "/$1/$2") - comment = imgurRe2.ReplaceAllString(comment, "/$1") + *comment = imgurRe.ReplaceAllString(*comment, "/$1/$2") + *comment = imgurRe2.ReplaceAllString(*comment, "/$1") p := bluemonday.UGCPolicy() p.AllowImages() @@ -122,24 +107,51 @@ func parseComment(data gjson.Result) Comment { p.RequireNoReferrerOnLinks(true) p.RequireNoFollowOnLinks(true) p.RequireCrossOriginAnonymous(true) - comment = p.Sanitize(comment) - - return Comment{ - Comments: comments, - User: User{ - Id: data.Get("account.id").Int(), - Username: data.Get("account.username").String(), - Avatar: userAvatar, - }, - Post: parseSubmission(data.Get("post")), - Id: data.Get("id").String(), - Comment: comment, - Upvotes: data.Get("upvote_count").Int(), - Downvotes: data.Get("downvote_count").Int(), - Platform: data.Get("platform").String(), - CreatedAt: createdAt, - RelTime: humanize.Time(createdTime), - UpdatedAt: updatedAt, - DeletedAt: deletedAt, - } + *comment = p.Sanitize(*comment) +} + +type commentArray []Comment + +func (arr *commentArray) UnmarshalJSON(data []byte) error { + var rawArr []json.RawMessage + err := json.Unmarshal(data, &rawArr) + if err != nil { + return err + } + *arr = make(commentArray, len(rawArr)) + panics := make([]error, 0, len(rawArr)) + wg := sync.WaitGroup{} + var handlePanic = func() { + if v := recover(); v != nil { + v, ok := v.(error) + if !ok { + v = fmt.Errorf("%v", v) + } + panics = append(panics, v) + } + } + for i, value := range rawArr { + wg.Add(1) + go func() { + defer handlePanic() + defer wg.Done() + parseComment(value, &(*arr)[i]) + }() + } + wg.Wait() + if len(panics) != 0 { + return errors.Join(panics...) + } + return nil +} + +type commentPost Submission + +func (post *commentPost) UnmarshalJSON(data []byte) error { + *post = commentPost(parseSubmission(gjson.Parse(string(data)))) + return nil +} + +type commentApiResponse struct { + Data commentArray `json:"data"` } diff --git a/api/user.go b/api/user.go index 576373f..886eb79 100644 --- a/api/user.go +++ b/api/user.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "io" "net/http" "strings" @@ -154,7 +155,7 @@ func (client *Client) FetchUserFavorites(username string, sort string, page stri func (client *Client) FetchUserComments(username string) ([]Comment, error) { cacheData, found := client.Cache.Get(username + "-usercomments") if found { - return cacheData.([]Comment), nil + return cacheData.(commentArray), nil } req, err := http.NewRequest("GET", "https://api.imgur.com/comment/v1/comments", nil) @@ -176,23 +177,19 @@ func (client *Client) FetchUserComments(username string) ([]Comment, error) { return []Comment{}, err } - body, err := io.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { return []Comment{}, err } - data := gjson.Parse(string(body)) + var parsed commentApiResponse + err = json.Unmarshal(data, &parsed) + if err != nil { + return []Comment{}, err + } - comments := make([]Comment, 0) - data.Get("data").ForEach( - func(key, value gjson.Result) bool { - comments = append(comments, parseComment(value)) - return true - }, - ) - - client.Cache.Set(username+"-usercomments", comments, cache.DefaultExpiration) - return comments, nil + client.Cache.Set(username+"-usercomments", parsed.Data, cache.DefaultExpiration) + return parsed.Data, nil } func parseSubmission(value gjson.Result) Submission { diff --git a/main.go b/main.go index 0b99d4a..505cdc0 100644 --- a/main.go +++ b/main.go @@ -23,13 +23,13 @@ func wrapHandler(h handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer func() { if v := recover(); v != nil { - utils.RenderError(w, r, 500, fmt.Sprint(v)) + pages.RenderError(w, r, 500, fmt.Sprint(v)) } }() err := h(w, r) if err != nil { fmt.Println(err) - utils.RenderError(w, r, 500, err.Error()) + pages.RenderError(w, r, 500, err.Error()) } }) } @@ -61,14 +61,14 @@ func main() { if os.Getenv("ENV") == "dev" { app.Handle("GET /errors/429", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - utils.RenderError(w, r, 429) + pages.RenderError(w, r, 429) })) app.Handle("GET /errors/429/img", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Location", "/static/img/error-429.png") w.WriteHeader(302) })) app.Handle("GET /errors/404", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - utils.RenderError(w, r, 404) + pages.RenderError(w, r, 404) })) app.Handle("GET /errors/404/img", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Location", "/static/img/error-404.png") diff --git a/pages/embed.go b/pages/embed.go index f7016c6..92c7694 100644 --- a/pages/embed.go +++ b/pages/embed.go @@ -24,10 +24,10 @@ func HandleEmbed(w http.ResponseWriter, r *http.Request) error { post, err = ApiClient.FetchMedia(r.PathValue("postID")) } if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil && post.Id == "" && strings.Contains(err.Error(), "404") { - return utils.RenderError(w, r, 404) + return RenderError(w, r, 404) } if err != nil { return err diff --git a/utils/error.go b/pages/error.go similarity index 87% rename from utils/error.go rename to pages/error.go index 30bd215..0806869 100644 --- a/utils/error.go +++ b/pages/error.go @@ -1,4 +1,4 @@ -package utils +package pages import ( "fmt" @@ -8,6 +8,7 @@ import ( "codeberg.org/rimgo/rimgo/render" "codeberg.org/rimgo/rimgo/static" + "codeberg.org/rimgo/rimgo/utils" ) func RenderError(w http.ResponseWriter, r *http.Request, code int, str ...string) (err error) { @@ -21,7 +22,7 @@ func RenderError(w http.ResponseWriter, r *http.Request, code int, str ...string if code != 500 { codeStr = strconv.Itoa(code) } - if !Accepts(r, "text/html") && r.PathValue("extension") != "" { + if !utils.Accepts(r, "text/html") && r.PathValue("extension") != "" { w.Header().Set("Content-Type", "image/png") w.WriteHeader(code) file, _ := static.GetFiles().Open("img/error-" + codeStr + ".png") diff --git a/pages/media.go b/pages/media.go index e327811..6480eaa 100644 --- a/pages/media.go +++ b/pages/media.go @@ -73,9 +73,9 @@ func handleMedia(w http.ResponseWriter, r *http.Request, url string) error { } if res.StatusCode == 404 || strings.Contains(res.Request.URL.String(), "error/404") { - return utils.RenderError(w, r, 404) + return RenderError(w, r, 404) } else if res.StatusCode == 429 { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } w.Header().Set("Accept-Ranges", "bytes") diff --git a/pages/post.go b/pages/post.go index 171d312..084f134 100644 --- a/pages/post.go +++ b/pages/post.go @@ -55,10 +55,10 @@ func HandlePost(w http.ResponseWriter, r *http.Request) error { post, err = ApiClient.FetchMedia(postId) } if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil && post.Id == "" && strings.Contains(err.Error(), "404") { - return utils.RenderError(w, r, 404) + return RenderError(w, r, 404) } if err != nil { return err diff --git a/pages/rss.go b/pages/rss.go index e748b40..d64f71f 100644 --- a/pages/rss.go +++ b/pages/rss.go @@ -78,7 +78,7 @@ func HandleUserRSS(w http.ResponseWriter, r *http.Request) error { submissions, err := ApiClient.FetchSubmissions(user, "newest", "1") if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil { return err diff --git a/pages/tag.go b/pages/tag.go index 4f1da58..abd7848 100644 --- a/pages/tag.go +++ b/pages/tag.go @@ -26,13 +26,13 @@ func HandleTag(w http.ResponseWriter, r *http.Request) error { tag, err := ApiClient.FetchTag(r.PathValue("tag"), r.URL.Query().Get("sort"), page) if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil { return err } if tag.Display == "" { - return utils.RenderError(w, r, 404) + return RenderError(w, r, 404) } return render.Render(w, "tag", map[string]any{ diff --git a/pages/user.go b/pages/user.go index 482b5d9..00aa387 100644 --- a/pages/user.go +++ b/pages/user.go @@ -26,18 +26,18 @@ func HandleUser(w http.ResponseWriter, r *http.Request) error { user, err := ApiClient.FetchUser(r.PathValue("userID")) if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil { return err } if user.Username == "" { - return utils.RenderError(w, r, 404) + return RenderError(w, r, 404) } submissions, err := ApiClient.FetchSubmissions(r.PathValue("userID"), "newest", page) if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil { return err @@ -60,18 +60,18 @@ func HandleUserComments(w http.ResponseWriter, r *http.Request) error { user, err := ApiClient.FetchUser(r.PathValue("userID")) if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil { return err } if user.Username == "" { - return utils.RenderError(w, r, 404) + return RenderError(w, r, 404) } comments, err := ApiClient.FetchUserComments(r.PathValue("userID")) if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil { return err @@ -101,18 +101,18 @@ func HandleUserFavorites(w http.ResponseWriter, r *http.Request) error { user, err := ApiClient.FetchUser(r.PathValue("userID")) if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil { return err } if user.Username == "" { - return utils.RenderError(w, r, 404) + return RenderError(w, r, 404) } favorites, err := ApiClient.FetchUserFavorites(r.PathValue("userID"), "newest", page) if err != nil && err.Error() == "ratelimited by imgur" { - return utils.RenderError(w, r, 429) + return RenderError(w, r, 429) } if err != nil { return err diff --git a/render/helpers.go b/render/helpers.go index a752ae3..7f7ba11 100644 --- a/render/helpers.go +++ b/render/helpers.go @@ -1,10 +1,19 @@ package render -import "github.com/mailgun/raymond/v2" +import ( + "time" + + "codeberg.org/rimgo/rimgo/utils" + "github.com/dustin/go-humanize" + "github.com/mailgun/raymond/v2" +) func (r *renderer) registerHelpers() { funcmap := map[string]any{ - "noteq": noteq, + "noteq": noteq, + "ifNonZeroTime": ifNonZeroTime, + "relTime": relTime, + "rewriteUrl": rewriteUrl, } raymond.RegisterHelpers(funcmap) } @@ -15,3 +24,19 @@ func noteq(a, b any, options *raymond.Options) any { } return "" } +func ifNonZeroTime(v any, options *raymond.Options) any { + if v.(time.Time).IsZero() { + return "" + } + return options.Fn() +} +func relTime(date time.Time) string { + return humanize.Time(date) +} +func rewriteUrl(link string) string { + r, err := utils.RewriteUrl(link) + if err != nil { + panic(err) + } + return r +} diff --git a/utils/request.go b/utils/request.go index 37f627a..50acaa5 100644 --- a/utils/request.go +++ b/utils/request.go @@ -1,6 +1,7 @@ package utils import ( + "encoding/json" "fmt" "io" "net/http" @@ -9,6 +10,42 @@ import ( "github.com/tidwall/gjson" ) +func GetJSONNew(url string) (json.RawMessage, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return json.RawMessage{}, err + } + + SetReqHeaders(req) + + client := http.Client{} + res, err := client.Do(req) + if err != nil { + return json.RawMessage{}, err + } + rateLimitRemaining := res.Header.Get("X-RateLimit-UserRemaining") + if rateLimitRemaining != "" { + ratelimit, _ := strconv.Atoi(rateLimitRemaining) + if ratelimit <= 0 { + return json.RawMessage{}, fmt.Errorf("ratelimited by imgur") + } + } + + body, err := io.ReadAll(res.Body) + if err != nil { + return json.RawMessage{}, err + } + + switch res.StatusCode { + case 200: + return body, nil + case 429: + return json.RawMessage{}, fmt.Errorf("ratelimited by imgur") + default: + return json.RawMessage{}, fmt.Errorf("received status %s, expected 200 OK.\n%s", res.Status, string(body)) + } +} + func GetJSON(url string) (gjson.Result, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/utils/rewriteUrl.go b/utils/rewriteUrl.go new file mode 100644 index 0000000..bc5ac1b --- /dev/null +++ b/utils/rewriteUrl.go @@ -0,0 +1,25 @@ +package utils + +import ( + "fmt" + "net/url" + "strings" +) + +func RewriteUrl(link string) (string, error) { + url, err := url.Parse(link) + if err != nil { + return "", err + } + path := url.Path + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + switch url.Host { + case "", "imgur.com", "www.imgur.com", "i.imgur.com": + return path, nil + case "i.stack.imgur.com": + return "/stack" + path, nil + } + return "", fmt.Errorf("unknown host %s", url.Host) +} diff --git a/views/partials/comment.hbs b/views/partials/comment.hbs index 476f868..9a84906 100644 --- a/views/partials/comment.hbs +++ b/views/partials/comment.hbs @@ -1,22 +1,22 @@
- {{#noteq this.User.Username "[deleted]"}} - - -

{{this.User.Username}}

+ {{#noteq this.Account.Username "[deleted]"}} + + +

{{this.Account.Username}}

 {{/noteq}} - {{#equal this.User.Username "[deleted]"}} + {{#equal this.Account.Username "[deleted]"}}

[deleted]

{{/equal}}

{{{this.Comment}}}

- {{this.RelTime}} - {{#if this.DeletedAt}} + {{relTime(this.CreatedAt)}} + {{#ifNonZeroTime this.DeletedAt}} (deleted {{this.DeletedAt}}) - {{/if}} + {{/ifNonZeroTime}} | Likes {{this.Upvotes}} Dislikes {{this.Downvotes}} diff --git a/views/partials/contextComment.hbs b/views/partials/contextComment.hbs index c5b0e43..5608901 100644 --- a/views/partials/contextComment.hbs +++ b/views/partials/contextComment.hbs @@ -6,10 +6,10 @@

{{{this.Comment}}}

- {{this.RelTime}} - {{#if this.DeletedAt}} + {{relTime(this.CreatedAt)}} + {{#ifNonZeroTime this.DeletedAt}} (deleted {{this.DeletedAt}}) - {{/if}} + {{/ifNonZeroTime}} | Likes {{this.Upvotes}}