添加rsshub #1203
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| // Package domain rsshub领域逻辑 | ||
| package domain | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "github.com/mmcdole/gofeed" | ||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| // syncRss 同步所有频道 | ||
| // 返回：更新的频道&订阅信息 map[int64]*RssClientView | ||
| // 1. 获取所有频道 | ||
| // 2. 遍历所有频道，检查频道是否更新 | ||
| // 3. 如果更新，获取更新的内容，但是返回的数据 | ||
| func (repo *rssDomain) syncRss(ctx context.Context) (updated map[int64]*RssClientView, err error) { | ||
| updated = make(map[int64]*RssClientView) | ||
| // 获取所有频道 | ||
| sources, err := repo.storage.GetSources(ctx) | ||
| if err != nil { | ||
| return | ||
| } | ||
| // 遍历所有源，获取每个channel对应的rss内容 | ||
| rssView := make([]*RssClientView, len(sources)) | ||
| for i, channel := range sources { | ||
| var feed *gofeed.Feed | ||
| // 从site获取rss内容 | ||
| feed, err = repo.rssHubClient.FetchFeed(channel.RssHubFeedPath) | ||
| // 如果获取失败，则跳过 | ||
| if err != nil { | ||
| logrus.WithContext(ctx).Errorf("[rsshub syncRss] fetch path(%+v) error: %v", channel.RssHubFeedPath, err) | ||
| continue | ||
| } | ||
| rv := convertFeedToRssView(0, channel.RssHubFeedPath, feed) | ||
| rssView[i] = rv | ||
| } | ||
| // 检查频道是否更新 | ||
| for _, cv := range rssView { | ||
| if cv == nil { | ||
| continue | ||
| } | ||
| var needUpdate bool | ||
| needUpdate, err = repo.checkSourceNeedUpdate(ctx, cv.Source) | ||
| if err != nil { | ||
| logrus.WithContext(ctx).Errorf("[rsshub syncRss] checkSourceNeedUpdate error: %v", err) | ||
| err = nil | ||
| continue | ||
| } | ||
| // 保存 | ||
| logrus.WithContext(ctx).Infof("[rsshub syncRss] cv %+v, need update(real): %v", cv.Source, needUpdate) | ||
| // 如果需要更新，更新channel 和 content | ||
| if needUpdate { | ||
| err = repo.storage.UpsertSource(ctx, cv.Source) | ||
| if err != nil { | ||
| logrus.WithContext(ctx).Errorf("[rsshub syncRss] upsert source error: %v", err) | ||
| err = nil | ||
| // continue | ||
| } | ||
| } | ||
| var updateChannelView = &RssClientView{Source: cv.Source, Contents: []*RssContent{}} | ||
| err = repo.processContentsUpdate(ctx, cv, err, updateChannelView) | ||
| if err != nil { | ||
| logrus.WithContext(ctx).Errorf("[rsshub syncRss] processContentsUpdate error: %v", err) | ||
| continue | ||
| } | ||
| if len(updateChannelView.Contents) == 0 { | ||
| logrus.WithContext(ctx).Infof("[rsshub syncRss] cv %s, no new content", cv.Source.RssHubFeedPath) | ||
| continue | ||
| } | ||
| updateChannelView.Sort() | ||
| updated[updateChannelView.Source.ID] = updateChannelView | ||
| logrus.WithContext(ctx).Debugf("[rsshub syncRss] cv %s, new contents: %v", cv.Source.RssHubFeedPath, len(updateChannelView.Contents)) | ||
| } | ||
| return | ||
| } | ||
|
|
||
| // checkSourceNeedUpdate 检查频道是否需要更新 | ||
| func (repo *rssDomain) checkSourceNeedUpdate(ctx context.Context, source *RssSource) (needUpdate bool, err error) { | ||
| var sourceInDB *RssSource | ||
| sourceInDB, err = repo.storage.GetSourceByRssHubFeedLink(ctx, source.RssHubFeedPath) | ||
| if err != nil { | ||
| return | ||
| } | ||
| if sourceInDB == nil { | ||
| logrus.WithContext(ctx).Errorf("[rsshub syncRss] source not found: %v", source.RssHubFeedPath) | ||
| return | ||
| } | ||
| source.ID = sourceInDB.ID | ||
| // 检查是否需要更新到db | ||
| if sourceInDB.IfNeedUpdate(source) { | ||
| needUpdate = true | ||
| } | ||
| return | ||
| } | ||
|
|
||
| // processContentsUpdate 处理内容(s)更新 | ||
| func (repo *rssDomain) processContentsUpdate(ctx context.Context, cv *RssClientView, err error, updateChannelView *RssClientView) error { | ||
| for _, content := range cv.Contents { | ||
| if content == nil { | ||
| continue | ||
| } | ||
| content.RssSourceID = cv.Source.ID | ||
| var existed bool | ||
| existed, err = repo.processContentItemUpdate(ctx, content) | ||
| if err != nil { | ||
| logrus.WithContext(ctx).Errorf("[rsshub syncRss] upsert content error: %v", err) | ||
| err = nil | ||
| continue | ||
| } | ||
| if !existed { | ||
| updateChannelView.Contents = append(updateChannelView.Contents, content) | ||
| logrus.WithContext(ctx).Infof("[rsshub syncRss] cv %s, add new content: %v", cv.Source.RssHubFeedPath, content.Title) | ||
| } | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| // processContentItemUpdate 处理单个内容更新 | ||
| func (repo *rssDomain) processContentItemUpdate(ctx context.Context, content *RssContent) (existed bool, err error) { | ||
| existed, err = repo.storage.IsContentHashIDExist(ctx, content.HashID) | ||
| if err != nil { | ||
| return | ||
| } | ||
| // 不需要更新&不需要发送 | ||
| if existed { | ||
| return | ||
| } | ||
| // 保存 | ||
| err = repo.storage.UpsertContent(ctx, content) | ||
| if err != nil { | ||
| logrus.WithContext(ctx).Errorf("[rsshub syncRss] upsert content error: %v", err) | ||
| return | ||
| } | ||
| return | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| package domain | ||
|
|
||
| import ( | ||
| "encoding/hex" | ||
| "hash/fnv" | ||
| "sort" | ||
| "time" | ||
| ) | ||
|
|
||
| // ======== RSS ========[START] | ||
|
|
||
| func genHashForFeedItem(link, guid string) string { | ||
| idString := link + "||" + guid | ||
| h := fnv.New32() | ||
| _, _ = h.Write([]byte(idString)) | ||
|
||
| encoded := hex.EncodeToString(h.Sum(nil)) | ||
| return encoded | ||
| } | ||
|
|
||
| // RssClientView 频道视图 | ||
| type RssClientView struct { | ||
| Source *RssSource | ||
| Contents []*RssContent | ||
| } | ||
|
|
||
| // ======== RSS ========[END] | ||
|
|
||
| // ======== DB ========[START] | ||
|
|
||
| const ( | ||
| tableNameRssSource = "rss_source" | ||
| tableNameRssContent = "rss_content" | ||
| tableNameRssSubscribe = "rss_subscribe" | ||
| ) | ||
|
|
||
| // RssSource RSS频道 | ||
| type RssSource struct { | ||
| // Id 自增id | ||
| ID int64 `gorm:"column:id;primary_key;AUTO_INCREMENT"` | ||
| // RssHubFeedPath 频道路由 用于区分rss_hub 不同的频道 例如: `/bangumi/tv/calendar/today` | ||
| RssHubFeedPath string `gorm:"column:rss_hub_feed_path;not null;unique;" json:"rss_hub_feed_path"` | ||
| // Title 频道标题 | ||
| Title string `gorm:"column:title" json:"title"` | ||
| // ChannelDesc 频道描述 | ||
| ChannelDesc string `gorm:"column:channel_desc" json:"channel_desc"` | ||
| // ImageURL 频道图片 | ||
| ImageURL string `gorm:"column:image_url" json:"image_url"` | ||
| // Link 频道链接 | ||
| Link string `gorm:"column:link" json:"link"` | ||
| // UpdatedParsed RSS页面更新时间 | ||
| UpdatedParsed time.Time `gorm:"column:updated_parsed" json:"updated_parsed"` | ||
| // Mtime update time | ||
| Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` | ||
| } | ||
|
|
||
| // TableName ... | ||
| func (RssSource) TableName() string { | ||
| return tableNameRssSource | ||
| } | ||
|
|
||
| // IfNeedUpdate ... | ||
| func (r RssSource) IfNeedUpdate(cmp *RssSource) bool { | ||
| if r.Link != cmp.Link { | ||
| return false | ||
| } | ||
| return r.UpdatedParsed.Unix() < cmp.UpdatedParsed.Unix() | ||
| } | ||
|
|
||
| // RssContent 订阅的RSS频道的推送信息 | ||
| type RssContent struct { | ||
| // Id 自增id | ||
| ID int64 `gorm:"column:id;primary_key;AUTO_INCREMENT"` | ||
| HashID string `gorm:"column:hash_id;unique" json:"hash_id"` | ||
| RssSourceID int64 `gorm:"column:rss_source_id;not null" json:"rss_source_id"` | ||
| Title string `gorm:"column:title" json:"title"` | ||
| Description string `gorm:"column:description" json:"description"` | ||
| Link string `gorm:"column:link" json:"link"` | ||
| Date time.Time `gorm:"column:date" json:"date"` | ||
| Author string `gorm:"column:author" json:"author"` | ||
| Thumbnail string `gorm:"column:thumbnail" json:"thumbnail"` | ||
| Content string `gorm:"column:content" json:"content"` | ||
| // Mtime update time | ||
| Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` | ||
| } | ||
|
|
||
| // TableName ... | ||
| func (RssContent) TableName() string { | ||
| return tableNameRssContent | ||
| } | ||
|
|
||
| // Sort ... order by Date desc | ||
| func (r *RssClientView) Sort() { | ||
| sort.Slice(r.Contents, func(i, j int) bool { | ||
| return r.Contents[i].Date.Unix() > r.Contents[j].Date.Unix() | ||
| }) | ||
| } | ||
|
|
||
| // RssSubscribe 订阅关系表：群组-RSS频道 | ||
| type RssSubscribe struct { | ||
| // Id 自增id | ||
| ID int64 `gorm:"column:id;primary_key;AUTO_INCREMENT"` | ||
| // 订阅群组 | ||
| GroupID int64 `gorm:"column:group_id;not null;uniqueIndex:uk_sid_gid"` | ||
| // 订阅频道 | ||
| RssSourceID int64 `gorm:"column:rss_source_id;not null;uniqueIndex:uk_sid_gid"` | ||
| // Mtime update time | ||
| Mtime time.Time `gorm:"column:mtime;default:current_timestamp;" json:"mtime"` | ||
| } | ||
|
|
||
| // TableName ... | ||
| func (RssSubscribe) TableName() string { | ||
| return tableNameRssSubscribe | ||
| } | ||
|
|
||
| // ======== DB ========[END] | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
这个err放在入参有什么用