mattburchett/NadekoBot
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

support for rss and atoms feeds added.

Browse Source
This commit is contained in:
Master Kwoth 2017-09-22 06:59:57 +02:00
parent e12c29dda5
commit 45b696bab8
16 changed files with 2469 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1985
src/NadekoBot/Migrations/20170921185313_feeds.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

59
src/NadekoBot/Migrations/20170921185313_feeds.cs Normal file
View File

@ -0,0 +1,59 @@
using Microsoft.EntityFrameworkCore.Migrations;
using System;
using System.Collections.Generic;
namespace NadekoBot.Migrations
{
public partial class feeds : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "FeedSub",
columns: table => new
{
Id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
ChannelId = table.Column<ulong>(type: "INTEGER", nullable: false),
DateAdded = table.Column<DateTime>(type: "TEXT", nullable: true),
GuildConfigId = table.Column<int>(type: "INTEGER", nullable: false),
Url = table.Column<string>(type: "TEXT", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_FeedSub", x => x.Id);
table.UniqueConstraint("AK_FeedSub_GuildConfigId_Url", x => new { x.GuildConfigId, x.Url });
table.ForeignKey(
name: "FK_FeedSub_GuildConfigs_GuildConfigId",
column: x => x.GuildConfigId,
principalTable: "GuildConfigs",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
});
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "FeedSub");
migrationBuilder.AlterColumn<DateTime>(
name: "LastLevelUp",
table: "UserXpStats",
nullable: false,
defaultValue: new DateTime(2017, 9, 15, 5, 48, 8, 665, DateTimeKind.Local),
oldClrType: typeof(DateTime),
oldType: "TEXT",
oldDefaultValue: new DateTime(2017, 9, 21, 20, 53, 13, 307, DateTimeKind.Local));
migrationBuilder.AlterColumn<DateTime>(
name: "LastLevelUp",
table: "DiscordUser",
nullable: false,
defaultValue: new DateTime(2017, 9, 15, 5, 48, 8, 660, DateTimeKind.Local),
oldClrType: typeof(DateTime),
oldType: "TEXT",
oldDefaultValue: new DateTime(2017, 9, 21, 20, 53, 13, 305, DateTimeKind.Local));
}
}
}

42
src/NadekoBot/Migrations/NadekoSqliteContextModelSnapshot.cs
View File

@ -1,10 +1,13 @@
using System;
// <auto-generated />
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.EntityFrameworkCore.Storage.Internal;
using NadekoBot.Services.Database;
using NadekoBot.Services.Database.Models;
using System;
namespace NadekoBot.Migrations
{
@ -13,8 +16,9 @@ namespace NadekoBot.Migrations
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "1.1.1");
.HasAnnotation("ProductVersion", "2.0.0-rtm-26452");
modelBuilder.Entity("NadekoBot.Services.Database.Models.AntiRaidSetting", b =>
{
@ -466,7 +470,7 @@ namespace NadekoBot.Migrations
b.Property<DateTime>("LastLevelUp")
.ValueGeneratedOnAdd()
.HasDefaultValue(new DateTime(2017, 9, 15, 5, 48, 8, 660, DateTimeKind.Local));
.HasDefaultValue(new DateTime(2017, 9, 21, 20, 53, 13, 305, DateTimeKind.Local));
b.Property<DateTime>("LastXpGain");
@ -546,6 +550,27 @@ namespace NadekoBot.Migrations
b.ToTable("ExcludedItem");
});
modelBuilder.Entity("NadekoBot.Services.Database.Models.FeedSub", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd();
b.Property<ulong>("ChannelId");
b.Property<DateTime?>("DateAdded");
b.Property<int>("GuildConfigId");
b.Property<string>("Url")
.IsRequired();
b.HasKey("Id");
b.HasAlternateKey("GuildConfigId", "Url");
b.ToTable("FeedSub");
});
modelBuilder.Entity("NadekoBot.Services.Database.Models.FilterChannelId", b =>
{
b.Property<int>("Id")
@ -1366,7 +1391,7 @@ namespace NadekoBot.Migrations
b.Property<DateTime>("LastLevelUp")
.ValueGeneratedOnAdd()
.HasDefaultValue(new DateTime(2017, 9, 15, 5, 48, 8, 665, DateTimeKind.Local));
.HasDefaultValue(new DateTime(2017, 9, 21, 20, 53, 13, 307, DateTimeKind.Local));
b.Property<int>("NotifyOnLevelUp");
@ -1693,6 +1718,14 @@ namespace NadekoBot.Migrations
.HasForeignKey("XpSettingsId");
});
modelBuilder.Entity("NadekoBot.Services.Database.Models.FeedSub", b =>
{
b.HasOne("NadekoBot.Services.Database.Models.GuildConfig", "GuildConfig")
.WithMany("FeedSubs")
.HasForeignKey("GuildConfigId")
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity("NadekoBot.Services.Database.Models.FilterChannelId", b =>
{
b.HasOne("NadekoBot.Services.Database.Models.GuildConfig")
@ -1945,6 +1978,7 @@ namespace NadekoBot.Migrations
.HasForeignKey("NadekoBot.Services.Database.Models.XpSettings", "GuildConfigId")
.OnDelete(DeleteBehavior.Cascade);
});
#pragma warning restore 612, 618
}
}
}

56
src/NadekoBot/Modules/Music/Services/MusicService.cs
View File

@ -222,61 +222,5 @@ namespace NadekoBot.Modules.Music.Services
if (MusicPlayers.TryRemove(id, out var mp))
await mp.Destroy();
}
//public Task<SongInfo> ResolveYoutubeSong(string query, string queuerName)
//{
// _log.Info("Getting video");
// //var (link, video) = await GetYoutubeVideo(query);
// //if (video == null) // do something with this error
// //{
// // _log.Info("Could not load any video elements based on the query.");
// // return null;
// //}
// ////var m = Regex.Match(query, @"\?t=(?<t>\d*)");
// ////int gotoTime = 0;
// ////if (m.Captures.Count > 0)
// //// int.TryParse(m.Groups["t"].ToString(), out gotoTime);
// //_log.Info("Creating song info");
// //var song = new SongInfo
// //{
// // Title = video.Title.Substring(0, video.Title.Length - 10), // removing trailing "- You Tube"
// // Provider = "YouTube",
// // Uri = async () => {
// // var vid = await GetYoutubeVideo(query);
// // if (vid.Item2 == null)
// // throw new HttpRequestException();
// // return await vid.Item2.GetUriAsync();
// // },
// // Query = link,
// // ProviderType = MusicType.YouTube,
// // QueuerName = queuerName
// //};
// return GetYoutubeVideo(query, queuerName);
//}
//private async Task<SongInfo> GetYoutubeVideo(string query, string queuerName)
//{
// //if (string.IsNullOrWhiteSpace(link))
// //{
// // _log.Info("No song found.");
// // return (null, null);
// //}
// //_log.Info("Getting all videos");
// //var allVideos = await Task.Run(async () => { try { return await _yt.GetAllVideosAsync(link).ConfigureAwait(false); } catch { return Enumerable.Empty<YouTubeVideo>(); } }).ConfigureAwait(false);
// //var videos = allVideos.Where(v => v.AdaptiveKind == AdaptiveKind.Audio);
// //var video = videos
// // .Where(v => v.AudioBitrate < 256)
// // .OrderByDescending(v => v.AudioBitrate)
// // .FirstOrDefault();
// //return (link, video);
//}
}
}

106
src/NadekoBot/Modules/Searches/FeedCommands.cs Normal file
View File

@ -0,0 +1,106 @@
using Discord;
using Discord.Commands;
using Discord.WebSocket;
using Microsoft.SyndicationFeed.Rss;
using NadekoBot.Common.Attributes;
using NadekoBot.Extensions;
using NadekoBot.Modules.Searches.Services;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Xml;
namespace NadekoBot.Modules.Searches
{
public partial class Searches
{
[Group]
public class FeedCommands : NadekoSubmodule<FeedsService>
{
private readonly DiscordSocketClient _client;
public FeedCommands(DiscordSocketClient client)
{
_client = client;
}
[NadekoCommand, Usage, Description, Aliases]
[RequireContext(ContextType.Guild)]
[RequireUserPermission(GuildPermission.ManageMessages)]
public async Task Feed(string url, [Remainder] ITextChannel channel = null)
{
var success = Uri.TryCreate(url, UriKind.Absolute, out var uri) &&
(uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps);
if (success)
{
channel = channel ?? (ITextChannel)Context.Channel;
using (var xmlReader = XmlReader.Create(url, new XmlReaderSettings() { Async = true }))
{
var reader = new RssFeedReader(xmlReader);
try
{
await reader.Read();
}
catch { success = false; }
}
if (success)
{
success = _service.AddFeed(Context.Guild.Id, channel.Id, url);
if (success)
{
await ReplyConfirmLocalized("feed_added").ConfigureAwait(false);
return;
}
}
}
await ReplyErrorLocalized("feed_not_valid").ConfigureAwait(false);
}
[NadekoCommand, Usage, Description, Aliases]
[RequireContext(ContextType.Guild)]
[RequireUserPermission(GuildPermission.ManageMessages)]
public async Task FeedRemove(int index)
{
if (_service.RemoveFeed(Context.Guild.Id, --index))
{
await ReplyConfirmLocalized("feed_removed").ConfigureAwait(false);
}
else
await ReplyErrorLocalized("feed_out_of_range").ConfigureAwait(false);
}
[NadekoCommand, Usage, Description, Aliases]
[RequireContext(ContextType.Guild)]
[RequireUserPermission(GuildPermission.ManageMessages)]
public async Task FeedList()
{
var feeds = _service.GetFeeds(Context.Guild.Id);
if (!feeds.Any())
{
await Context.Channel.EmbedAsync(new EmbedBuilder()
.WithOkColor()
.WithDescription(GetText("feed_no_feed")))
.ConfigureAwait(false);
return;
}
await Context.Channel.SendPaginatedConfirmAsync(_client, 0, (cur) =>
{
var embed = new EmbedBuilder()
.WithOkColor();
var i = 0;
var fs = string.Join("\n", feeds.Skip(cur * 10)
.Take(10)
.Select(x => $"`{(cur * 10) + (++i)}.` <#{x.ChannelId}> {x.Url}"));
return embed.WithDescription(fs);
}, feeds.Count / 10);
}
}
}
}

213
src/NadekoBot/Modules/Searches/Services/FeedsService.cs Normal file
View File

@ -0,0 +1,213 @@
using Discord;
using Microsoft.SyndicationFeed;
using Microsoft.SyndicationFeed.Rss;
using NadekoBot.Extensions;
using NadekoBot.Services;
using System;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml;
using System.Collections.Generic;
using NadekoBot.Services.Database.Models;
using Microsoft.EntityFrameworkCore;
using System.Collections.Concurrent;
using Discord.WebSocket;
namespace NadekoBot.Modules.Searches.Services
{
public class FeedsService : INService
{
private readonly DbService _db;
private readonly ConcurrentDictionary<string, HashSet<FeedSub>> _subs;
private readonly DiscordSocketClient _client;
private readonly ConcurrentDictionary<string, DateTime> _lastPosts =
new ConcurrentDictionary<string, DateTime>();
public FeedsService(IEnumerable<GuildConfig> gcs, DbService db, DiscordSocketClient client)
{
_db = db;
_subs = gcs.SelectMany(x => x.FeedSubs)
.GroupBy(x => x.Url)
.ToDictionary(x => x.Key, x => x.ToHashSet())
.ToConcurrent();
_client = client;
foreach (var kvp in _subs)
{
// to make sure rss feeds don't post right away, but
// only the updates from after the bot has started
_lastPosts.AddOrUpdate(kvp.Key, DateTime.UtcNow, (k, old) => DateTime.UtcNow);
}
var _ = Task.Run(TrackFeeds);
}
public async Task<EmbedBuilder> TrackFeeds()
{
while (true)
{
foreach (var kvp in _subs)
{
if (kvp.Value.Count == 0)
continue;
DateTime lastTime;
if (!_lastPosts.TryGetValue(kvp.Key, out lastTime))
lastTime = _lastPosts.AddOrUpdate(kvp.Key, DateTime.UtcNow, (k, old) => DateTime.UtcNow);
var rssUrl = kvp.Key;
try
{
using (var xmlReader = XmlReader.Create(rssUrl, new XmlReaderSettings() { Async = true }))
{
var feedReader = new RssFeedReader(xmlReader);
var embed = new EmbedBuilder()
.WithAuthor(kvp.Key)
.WithOkColor();
while (await feedReader.Read() && feedReader.ElementType != SyndicationElementType.Item)
{
switch (feedReader.ElementType)
{
case SyndicationElementType.Link:
var uri = await feedReader.ReadLink();
embed.WithAuthor(kvp.Key, url: uri.Uri.AbsoluteUri);
break;
case SyndicationElementType.Content:
var content = await feedReader.ReadContent();
break;
case SyndicationElementType.Category:
break;
case SyndicationElementType.Image:
ISyndicationImage image = await feedReader.ReadImage();
embed.WithThumbnailUrl(image.Url.AbsoluteUri);
break;
default:
break;
}
}
ISyndicationItem item = await feedReader.ReadItem();
if (item.Published.UtcDateTime <= lastTime)
continue;
var desc = item.Description.StripHTML();
lastTime = item.Published.UtcDateTime;
var title = string.IsNullOrWhiteSpace(item.Title) ? "-" : item.Title;
desc = Format.Code(item.Published.ToString()) + Environment.NewLine + desc;
var link = item.Links.FirstOrDefault();
if (link != null)
desc = $"[link]({link.Uri}) " + desc;
var img = item.Links.FirstOrDefault(x => x.RelationshipType == "enclosure")?.Uri.AbsoluteUri
?? Regex.Match(item.Description, @"src=""(?<src>.*?)""").Groups["src"].ToString();
if (!string.IsNullOrWhiteSpace(img) && Uri.IsWellFormedUriString(img, UriKind.Absolute))
embed.WithImageUrl(img);
embed.AddField(title, desc);
//send the created embed to all subscribed channels
var sendTasks = kvp.Value
.Where(x => x.GuildConfig != null)
.Select(x => _client.GetGuild(x.GuildConfig.GuildId)
?.GetTextChannel(x.ChannelId))
.Where(x => x != null)
.Select(x => x.EmbedAsync(embed));
_lastPosts.AddOrUpdate(kvp.Key, item.Published.UtcDateTime, (k, old) => item.Published.UtcDateTime);
await Task.WhenAll(sendTasks).ConfigureAwait(false);
}
}
catch (Exception ex) { Console.WriteLine(ex); }
}
await Task.Delay(10000);
}
}
public List<FeedSub> GetFeeds(ulong guildId)
{
using (var uow = _db.UnitOfWork)
{
return uow.GuildConfigs.For(guildId, set => set.Include(x => x.FeedSubs))
.FeedSubs
.OrderBy(x => x.Id)
.ToList();
}
}
public bool AddFeed(ulong guildId, ulong channelId, string rssFeed)
{
rssFeed.ThrowIfNull(nameof(rssFeed));
var fs = new FeedSub()
{
ChannelId = channelId,
Url = rssFeed.Trim().ToLowerInvariant(),
};
using (var uow = _db.UnitOfWork)
{
var gc = uow.GuildConfigs.For(guildId, set => set.Include(x => x.FeedSubs));
if (gc.FeedSubs.Contains(fs))
{
return false;
}
else if (gc.FeedSubs.Count >= 10)
{
return false;
}
gc.FeedSubs.Add(fs);
//adding all, in case bot wasn't on this guild when it started
foreach (var f in gc.FeedSubs)
{
_subs.AddOrUpdate(f.Url, new HashSet<FeedSub>(), (k, old) =>
{
old.Add(f);
return old;
});
}
uow.Complete();
}
return true;
}
public bool RemoveFeed(ulong guildId, int index)
{
if (index < 0)
return false;
using (var uow = _db.UnitOfWork)
{
var items = uow.GuildConfigs.For(guildId, set => set.Include(x => x.FeedSubs))
.FeedSubs
.OrderBy(x => x.Id)
.ToList();
if (items.Count <= index)
return false;
var toRemove = items[index];
_subs.AddOrUpdate(toRemove.Url, new HashSet<FeedSub>(), (key, old) =>
{
old.Remove(toRemove);
return old;
});
uow._context.Remove(toRemove);
uow.Complete();
}
return true;
}
}
}

1
src/NadekoBot/NadekoBot.csproj
View File

@ -61,6 +61,7 @@
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="2.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.0.0" />
<PackageReference Include="Microsoft.SyndicationFeed.ReaderWriter" Version="1.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
<PackageReference Include="NLog" Version="5.0.0-beta10" />
<PackageReference Include="StackExchange.Redis" Version="1.2.6" />

4
src/NadekoBot/Properties/launchSettings.json
View File

@ -2,6 +2,10 @@
"profiles": {
"NadekoBot": {
"commandName": "Project"
},
"Watch": {
"executablePath": "C:\\Program Files\\dotnet\\dotnet.exe",
"commandLineArgs": "watch run"
}
}
}

23
src/NadekoBot/Services/Database/Models/FeedSub.cs Normal file
View File

@ -0,0 +1,23 @@
namespace NadekoBot.Services.Database.Models
{
public class FeedSub : DbEntity
{
public int GuildConfigId { get; set; }
public GuildConfig GuildConfig { get; set; }
public ulong ChannelId { get; set; }
public string Url { get; set; }
public override int GetHashCode()
{
return Url.GetHashCode() ^ GuildConfigId.GetHashCode();
}
public override bool Equals(object obj)
{
return obj is FeedSub s
? s.Url == Url && s.GuildConfigId == GuildConfigId
: false;
}
}
}

1
src/NadekoBot/Services/Database/Models/GuildConfig.cs
View File

@ -87,6 +87,7 @@ namespace NadekoBot.Services.Database.Models
public StreamRoleSettings StreamRole { get; set; }
public XpSettings XpSettings { get; set; }
public List<FeedSub> FeedSubs { get; set; } = new List<FeedSub>();
//public List<ProtectionIgnoredChannel> ProtectionIgnoredChannels { get; set; } = new List<ProtectionIgnoredChannel>();
}

4
src/NadekoBot/Services/Database/NadekoContext.cs
View File

@ -142,6 +142,9 @@ namespace NadekoBot.Services.Database
.HasOne(x => x.GuildConfig)
.WithOne(x => x.AntiRaidSetting);
modelBuilder.Entity<FeedSub>()
.HasAlternateKey(x => new { x.GuildConfigId, x.Url });
//modelBuilder.Entity<ProtectionIgnoredChannel>()
// .HasAlternateKey(c => new { c.ChannelId, c.ProtectionType });
@ -304,6 +307,7 @@ namespace NadekoBot.Services.Database
.WithOne(x => x.XpSettings);
#endregion
//todo major bug
#region XpRoleReward
modelBuilder.Entity<XpRoleReward>()
.HasAlternateKey(x => x.Level);

2
src/NadekoBot/Services/Database/Repositories/Impl/GuildConfigRepository.cs
View File

@ -44,6 +44,8 @@ namespace NadekoBot.Services.Database.Repositories.Impl
.Include(gc => gc.SlowmodeIgnoredUsers)
.Include(gc => gc.AntiSpamSetting)
.ThenInclude(x => x.IgnoredChannels)
.Include(gc => gc.FeedSubs)
.ThenInclude(x => x.GuildConfig)
.Include(gc => gc.FollowedStreams)
.Include(gc => gc.StreamRole)
.Include(gc => gc.NsfwBlacklistedTags)

5
src/NadekoBot/_Extensions/StringExtensions.cs
View File

@ -9,6 +9,11 @@ namespace NadekoBot.Extensions
{
public static class StringExtensions
{
public static string StripHTML(this string input)
{
return Regex.Replace(input, "<.*?>", String.Empty);
}
/// <summary>
/// Easy use of fast, efficient case-insensitive Contains check with StringComparison Member Types
/// CurrentCulture, CurrentCultureIgnoreCase, InvariantCulture, InvariantCultureIgnoreCase, Ordinal, OrdinalIgnoreCase

7
src/NadekoBot/_strings/ResponseStrings.en-US.json
View File

@ -872,5 +872,10 @@
"xp_club_admin_remove": "{0} is no longer club admin.",
"xp_club_admin_error": "Error. You are either not the owner of the club, or that user is not in your club.",
"nsfw_started": "Started. Reposting every {0}s.",
"nsfw_stopped": "Stopped reposting."
"nsfw_stopped": "Stopped reposting.",
"searches_feed_added": "Feed added.",
"searches_feed_not_valid": "Invalid link, or you're already following that feed on this server, or you've reached maximum number of feeds allowed.",
"searches_feed_out_of_range": "Index out of range.",
"searches_feed_removed": "Feed removed.",
"searches_feed_no_feed": "You haven't subscribed to any feeds on this server."
}

21
src/NadekoBot/data/command_strings.json
View File

@ -2965,5 +2965,26 @@
"Usage": [
"{0}8ball"
]
},
"feed": {
"cmd": "feed feedadd",
"desc": "Subscribes to a feed. Bot will post an update up to once every 10 seconds. You can have up to 10 feeds on one server. All feeds must have unique URLs.",
"usage": [
"{0}feed https://www.rt.com/rss/"
]
},
"feedremove": {
"cmd": "feedremove feedrm feeddel",
"desc": "Stops tracking a feed on the given index. Use `{0}feeds` command to see a list of feeds and their indexes.",
"usage": [
"{0}feedremove 3"
]
},
"feedlist": {
"cmd": "feeds feedlist",
"desc": "Shows the list of feeds you've subscribed to on this server.",
"usage": [
"{0}feeds"
]
}
}