Merge remote-tracking branch 'refs/remotes/Kwoth/1.4' into 1.4

src/NadekoBot/Modules/Administration/Administration.cs

@@ -9,7 +9,6 @@ using NadekoBot.Services;
 using NadekoBot.Attributes;
 using NadekoBot.Services.Database.Models;
 using NadekoBot.Services.Administration;
-using Discord.WebSocket;
 
 namespace NadekoBot.Modules.Administration
 {
@@ -311,67 +310,6 @@ namespace NadekoBot.Modules.Administration
             await ReplyConfirmLocalized("set_channel_name").ConfigureAwait(false);
         }
 
-
-        //delets her own messages, no perm required
-        [NadekoCommand, Usage, Description, Aliases]
-        [RequireContext(ContextType.Guild)]
-        public async Task Prune()
-        {
-            var user = await Context.Guild.GetCurrentUserAsync().ConfigureAwait(false);
-
-            var enumerable = (await Context.Channel.GetMessagesAsync().Flatten())
-                .Where(x => x.Author.Id == user.Id && DateTime.UtcNow - x.CreatedAt < twoWeeks);
-            await Context.Channel.DeleteMessagesAsync(enumerable).ConfigureAwait(false);
-            Context.Message.DeleteAfter(3);
-        }
-
-
-        private TimeSpan twoWeeks => TimeSpan.FromDays(14);
-        // prune x
-        [NadekoCommand, Usage, Description, Aliases]
-        [RequireContext(ContextType.Guild)]
-        [RequireUserPermission(ChannelPermission.ManageMessages)]
-        [RequireBotPermission(GuildPermission.ManageMessages)]
-        [Priority(0)]
-        public async Task Prune(int count)
-        {
-            if (count < 1)
-                return;
-            await Context.Message.DeleteAsync().ConfigureAwait(false);
-            int limit = (count < 100) ? count + 1 : 100;
-            var enumerable = (await Context.Channel.GetMessagesAsync(limit: limit).Flatten().ConfigureAwait(false))
-                .Where(x => DateTime.UtcNow - x.CreatedAt < twoWeeks);
-            if (enumerable.FirstOrDefault()?.Id == Context.Message.Id)
-                enumerable = enumerable.Skip(1).ToArray();
-            else
-                enumerable = enumerable.Take(count);
-            await Context.Channel.DeleteMessagesAsync(enumerable).ConfigureAwait(false);
-        }
-
-        //prune @user [x]
-        [NadekoCommand, Usage, Description, Aliases]
-        [RequireContext(ContextType.Guild)]
-        [RequireUserPermission(ChannelPermission.ManageMessages)]
-        [RequireBotPermission(GuildPermission.ManageMessages)]
-        [Priority(1)]
-        public async Task Prune(IGuildUser user, int count = 100)
-        {
-            if (count < 1)
-                return;
-
-            if (count > 100)
-                count = 100;
-
-            if (user.Id == Context.User.Id)
-                count += 1;
-            var enumerable = (await Context.Channel.GetMessagesAsync().Flatten())
-                .Where(m => m.Author.Id == user.Id && DateTime.UtcNow - m.CreatedAt < twoWeeks)
-                .Take(count);
-            await Context.Channel.DeleteMessagesAsync(enumerable).ConfigureAwait(false);
-
-            Context.Message.DeleteAfter(3);
-        }
-
         [NadekoCommand, Usage, Description, Aliases]
         [RequireContext(ContextType.Guild)]
         [RequireUserPermission(GuildPermission.MentionEveryone)]

src/NadekoBot/Modules/Administration/Commands/PruneCommands.cs

@@ -0,0 +1,73 @@
+using Discord;
+using Discord.Commands;
+using NadekoBot.Attributes;
+using NadekoBot.Extensions;
+using NadekoBot.Services.Administration;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NadekoBot.Modules.Administration
+{
+    public partial class Administration
+    {
+        [Group]
+        public class PruneCommands : ModuleBase
+        {
+            private readonly TimeSpan twoWeeks = TimeSpan.FromDays(14);
+            private readonly PruneService _prune;
+
+            public PruneCommands(PruneService prune)
+            {
+                _prune = prune;
+            }
+
+            //delets her own messages, no perm required
+            [NadekoCommand, Usage, Description, Aliases]
+            [RequireContext(ContextType.Guild)]
+            public async Task Prune()
+            {
+                var user = await Context.Guild.GetCurrentUserAsync().ConfigureAwait(false);
+
+                await _prune.PruneWhere((ITextChannel)Context.Channel, 100, (x) => x.Author.Id == user.Id).ConfigureAwait(false);
+                Context.Message.DeleteAfter(3);
+            }
+            // prune x
+            [NadekoCommand, Usage, Description, Aliases]
+            [RequireContext(ContextType.Guild)]
+            [RequireUserPermission(ChannelPermission.ManageMessages)]
+            [RequireBotPermission(GuildPermission.ManageMessages)]
+            [Priority(0)]
+            public async Task Prune(int count)
+            {
+                count++;
+                if (count < 1)
+                    return;
+                if (count > 1000)
+                    count = 1000;
+                await _prune.PruneWhere((ITextChannel)Context.Channel, count, x => true).ConfigureAwait(false);
+            }
+
+            //prune @user [x]
+            [NadekoCommand, Usage, Description, Aliases]
+            [RequireContext(ContextType.Guild)]
+            [RequireUserPermission(ChannelPermission.ManageMessages)]
+            [RequireBotPermission(GuildPermission.ManageMessages)]
+            [Priority(1)]
+            public async Task Prune(IGuildUser user, int count = 100)
+            {
+                if (user.Id == Context.User.Id)
+                    count++;
+
+                if (count < 1)
+                    return;
+
+                if (count > 1000)
+                    count = 1000;
+                await _prune.PruneWhere((ITextChannel)Context.Channel, count, m => m.Author.Id == user.Id && DateTime.UtcNow - m.CreatedAt < twoWeeks);
+            }
+        }
+    }
+}

src/NadekoBot/Modules/NadekoModule.cs

@@ -121,8 +121,10 @@ namespace NadekoBot.Modules
                         return Task.CompletedTask;
                     }
 
-                    userInputTask.SetResult(arg.Content);
+                    if (userInputTask.TrySetResult(arg.Content))
-                    userMsg.DeleteAfter(1);
+                    {
+                        userMsg.DeleteAfter(1);
+                    }
                     return Task.CompletedTask;
                 });
                 return Task.CompletedTask;

src/NadekoBot/NadekoBot.cs

@@ -86,13 +86,13 @@ namespace NadekoBot
                 LogLevel = LogSeverity.Warning,
                 TotalShards = Credentials.TotalShards,
                 ConnectionTimeout = int.MaxValue,
-                AlwaysDownloadUsers = true,
+                AlwaysDownloadUsers = false,
             });
 
             CommandService = new CommandService(new CommandServiceConfig()
             {
                 CaseSensitiveCommands = false,
-                DefaultRunMode = RunMode.Sync,
+                DefaultRunMode = RunMode.Async,
             });
 
             //foundation services
@@ -127,6 +127,7 @@ namespace NadekoBot
             var commandMapService = new CommandMapService(AllGuildConfigs);
             var patreonRewardsService = new PatreonRewardsService(Credentials, Db, Currency);
             var verboseErrorsService = new VerboseErrorsService(AllGuildConfigs, Db, CommandHandler, helpService);
+            var pruneService = new PruneService();
             #endregion
 
             #region permissions
@@ -197,6 +198,7 @@ namespace NadekoBot
                     .Add(converterService)
                     .Add(verboseErrorsService)
                     .Add(patreonRewardsService)
+                    .Add(pruneService)
                 .Add<SearchesService>(searchesService)
                     .Add(streamNotificationService)
                     .Add(animeSearchService)
@@ -247,14 +249,12 @@ namespace NadekoBot
             await Client.LoginAsync(TokenType.Bot, token).ConfigureAwait(false);
             await Client.StartAsync().ConfigureAwait(false);
 
-            // wait for all shards to be ready
-            int readyCount = 0;
-            foreach (var s in Client.Shards)
-                s.Ready += () => Task.FromResult(Interlocked.Increment(ref readyCount));
-
             _log.Info("Waiting for all shards to connect...");
-            while (readyCount < Client.Shards.Count)
+            while (!Client.Shards.All(x => x.ConnectionState == ConnectionState.Connected))
-                await Task.Delay(100).ConfigureAwait(false);
+            {
+                _log.Info("Connecting... {0}/{1}", Client.Shards.Count(x => x.ConnectionState == ConnectionState.Connected), Client.Shards.Count);
+                await Task.Delay(1000).ConfigureAwait(false);
+            }
         }
 
         public async Task RunAsync(params string[] args)

src/NadekoBot/Services/Administration/PruneService.cs

@@ -0,0 +1,70 @@
+using Discord;
+using NadekoBot.Extensions;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NadekoBot.Services.Administration
+{
+    public class PruneService
+    {
+        //channelids where prunes are currently occuring
+        private ConcurrentHashSet<ulong> _pruningChannels = new ConcurrentHashSet<ulong>();
+        private readonly TimeSpan twoWeeks = TimeSpan.FromDays(14);
+
+        public async Task PruneWhere(ITextChannel channel, int amount, Func<IMessage, bool> predicate)
+        {
+            channel.ThrowIfNull(nameof(channel));
+            if (amount <= 0)
+                throw new ArgumentOutOfRangeException(nameof(amount));
+
+            if (!_pruningChannels.Add(channel.Id))
+                return;
+
+            try
+            {
+                IMessage[] msgs;
+                IMessage lastMessage = null;
+                msgs = (await channel.GetMessagesAsync().Flatten()).Where(predicate).Take(amount).ToArray();
+                while (amount > 0 && msgs.Any())
+                {
+                    lastMessage = msgs[msgs.Length - 1];
+
+                    var bulkDeletable = new List<IMessage>();
+                    var singleDeletable = new List<IMessage>();
+                    foreach (var x in msgs)
+                    {
+                        if (DateTime.UtcNow - x.CreatedAt < twoWeeks)
+                            bulkDeletable.Add(x);
+                        else
+                            singleDeletable.Add(x);
+                    }
+
+                    if (bulkDeletable.Count > 0)
+                        await Task.WhenAll(Task.Delay(1000), channel.DeleteMessagesAsync(bulkDeletable)).ConfigureAwait(false);
+
+                    var i = 0;
+                    foreach (var group in singleDeletable.GroupBy(x => ++i / (singleDeletable.Count / 5)))
+                        await Task.WhenAll(Task.Delay(1000), Task.WhenAll(group.Select(x => x.DeleteAsync()))).ConfigureAwait(false);
+
+                    //this isn't good, because this still work as if i want to remove only specific user's messages from the last
+                    //100 messages, Maybe this needs to be reduced by msgs.Length instead of 100
+                    amount -= 100;
+                    if(amount > 0)
+                        msgs = (await channel.GetMessagesAsync(lastMessage, Direction.Before).Flatten()).Where(predicate).Take(amount).ToArray();
+                }
+            }
+            catch
+            {
+                //ignore
+            }
+            finally
+            {
+                _pruningChannels.TryRemove(channel.Id);
+            }
+        }
+    }
+}

src/NadekoBot/Services/CustomReactions/Extensions.cs

@@ -1,10 +1,13 @@
-using Discord;
+using AngleSharp;
+using AngleSharp.Dom.Html;
+using Discord;
 using Discord.WebSocket;
 using NadekoBot.DataStructures;
 using NadekoBot.Extensions;
 using NadekoBot.Services.Database.Models;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 
@@ -14,13 +17,7 @@ namespace NadekoBot.Services.CustomReactions
     {
         public static Dictionary<string, Func<IUserMessage, string, string>> responsePlaceholders = new Dictionary<string, Func<IUserMessage, string, string>>()
         {
-            {"%target%", (ctx, trigger) => { return ctx.Content.Substring(trigger.Length).Trim().SanitizeMentions(); } }
+            {"%target%", (ctx, trigger) => { return ctx.Content.Substring(trigger.Length).Trim().SanitizeMentions(); } },
-        };
-
-        public static Dictionary<string, Func<IUserMessage, DiscordShardedClient, string>> placeholders = new Dictionary<string, Func<IUserMessage, DiscordShardedClient, string>>()
-        {
-            {"%mention%", (ctx, client) => { return $"<@{client.CurrentUser.Id}>"; } },
-            {"%user%", (ctx, client) => { return ctx.Author.Mention; } },
             {"%rnduser%", (ctx, client) => {
                 //var ch = ctx.Channel as ITextChannel;
                 //if(ch == null)
@@ -40,15 +37,22 @@ namespace NadekoBot.Services.CustomReactions
                 //var users = g.Users.ToArray();
 
                 //return users[new NadekoRandom().Next(0, users.Length-1)].Mention;
-            } }
+            } },
+        };
+
+        public static Dictionary<string, Func<IUserMessage, DiscordShardedClient, string>> placeholders = new Dictionary<string, Func<IUserMessage, DiscordShardedClient, string>>()
+        {
+            {"%mention%", (ctx, client) => { return $"<@{client.CurrentUser.Id}>"; } },
+            {"%user%", (ctx, client) => { return ctx.Author.Mention; } },
             //{"%rng%", (ctx) => { return new NadekoRandom().Next(0,10).ToString(); } }
         };
 
         private static readonly Regex rngRegex = new Regex("%rng(?:(?<from>(?:-)?\\d+)-(?<to>(?:-)?\\d+))?%", RegexOptions.Compiled);
+        private static readonly Regex imgRegex = new Regex("%(img|image):(?<tag>.*?)%", RegexOptions.Compiled);
 
         private static readonly NadekoRandom rng = new NadekoRandom();
 
-        public static Dictionary<Regex, MatchEvaluator> regexPlaceholders = new Dictionary<Regex, MatchEvaluator>()
+        public static Dictionary<Regex, Func<Match, Task<string>>> regexPlaceholders = new Dictionary<Regex, Func<Match, Task<string>>>()
         {
             { rngRegex, (match) => {
                 int from = 0;
@@ -59,13 +63,34 @@ namespace NadekoBot.Services.CustomReactions
 
                 if(from == 0 && to == 0)
                 {
-                    return rng.Next(0, 11).ToString();
+                    return Task.FromResult(rng.Next(0, 11).ToString());
                 }
 
                 if(from >= to)
+                    return Task.FromResult(string.Empty);
+
+                return Task.FromResult(rng.Next(from,to+1).ToString());
+            } },
+            { imgRegex, async (match) => {
+                var tag = match.Groups["tag"].ToString();
+                if(string.IsNullOrWhiteSpace(tag))
                     return "";
 
-                return rng.Next(from,to+1).ToString();
+                var fullQueryLink = $"http://imgur.com/search?q={ tag }";
+                var config = Configuration.Default.WithDefaultLoader();
+                var document = await BrowsingContext.New(config).OpenAsync(fullQueryLink);
+
+                var elems = document.QuerySelectorAll("a.image-list-link").ToArray();
+
+                if (!elems.Any())
+                    return "";
+
+                var img = (elems.ElementAtOrDefault(new NadekoRandom().Next(0, elems.Length))?.Children?.FirstOrDefault() as IHtmlImageElement);
+
+                if (img?.Source == null)
+                    return "";
+
+                return " "+img.Source.Replace("b.", ".") + " ";
             } }
         };
 
@@ -73,26 +98,31 @@ namespace NadekoBot.Services.CustomReactions
         {
             foreach (var ph in placeholders)
             {
-                str = str.ToLowerInvariant().Replace(ph.Key, ph.Value(ctx, client));
+                if (str.Contains(ph.Key))
+                    str = str.ToLowerInvariant().Replace(ph.Key, ph.Value(ctx, client));
             }
             return str;
         }
 
-        private static string ResolveResponseString(this string str, IUserMessage ctx, DiscordShardedClient client, string resolvedTrigger)
+        private static async Task<string> ResolveResponseStringAsync(this string str, IUserMessage ctx, DiscordShardedClient client, string resolvedTrigger)
         {
             foreach (var ph in placeholders)
             {
-                str = str.Replace(ph.Key.ToLowerInvariant(), ph.Value(ctx, client));
+                var lowerKey = ph.Key.ToLowerInvariant();
+                if (str.Contains(lowerKey))
+                    str = str.Replace(lowerKey, ph.Value(ctx, client));
             }
 
             foreach (var ph in responsePlaceholders)
             {
-                str = str.Replace(ph.Key.ToLowerInvariant(), ph.Value(ctx, resolvedTrigger));
+                var lowerKey = ph.Key.ToLowerInvariant();
+                if (str.Contains(lowerKey))
+                    str = str.Replace(lowerKey, ph.Value(ctx, resolvedTrigger));
             }
 
             foreach (var ph in regexPlaceholders)
             {
-                str = ph.Key.Replace(str, ph.Value);
+                str = await ph.Key.ReplaceAsync(str, ph.Value);
             }
             return str;
         }
@@ -100,8 +130,8 @@ namespace NadekoBot.Services.CustomReactions
         public static string TriggerWithContext(this CustomReaction cr, IUserMessage ctx, DiscordShardedClient client)
             => cr.Trigger.ResolveTriggerString(ctx, client);
 
-        public static string ResponseWithContext(this CustomReaction cr, IUserMessage ctx, DiscordShardedClient client)
+        public static Task<string > ResponseWithContextAsync(this CustomReaction cr, IUserMessage ctx, DiscordShardedClient client)
-            => cr.Response.ResolveResponseString(ctx, client, cr.Trigger.ResolveTriggerString(ctx, client));
+            => cr.Response.ResolveResponseStringAsync(ctx, client, cr.Trigger.ResolveTriggerString(ctx, client));
 
         public static async Task<IUserMessage> Send(this CustomReaction cr, IUserMessage context, DiscordShardedClient client, CustomReactionsService crs)
         {
@@ -113,7 +143,7 @@ namespace NadekoBot.Services.CustomReactions
             {
                 return await channel.EmbedAsync(crembed.ToEmbed(), crembed.PlainText ?? "");
             }
-            return await channel.SendMessageAsync(cr.ResponseWithContext(context, client).SanitizeMentions());
+            return await channel.SendMessageAsync((await cr.ResponseWithContextAsync(context, client)).SanitizeMentions());
         }
     }
 }

src/NadekoBot/_Extensions/Extensions.cs

@@ -11,6 +11,7 @@ using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Security.Cryptography;
+using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 
@@ -18,6 +19,23 @@ namespace NadekoBot.Extensions
 {
     public static class Extensions
     {
+        public static async Task<string> ReplaceAsync(this Regex regex, string input, Func<Match, Task<string>> replacementFn)
+        {
+            var sb = new StringBuilder();
+            var lastIndex = 0;
+
+            foreach (Match match in regex.Matches(input))
+            {
+                sb.Append(input, lastIndex, match.Index - lastIndex)
+                  .Append(await replacementFn(match).ConfigureAwait(false));
+
+                lastIndex = match.Index + match.Length;
+            }
+
+            sb.Append(input, lastIndex, input.Length - lastIndex);
+            return sb.ToString();
+        }
+
         public static void ThrowIfNull<T>(this T obj, string name) where T : class
         {
             if (obj == null)