Skip to content

Commit 035c55f

Browse files
nothingTVatYTnothingTVatYT
committed
add scanner for plugin details
1 parent 8c90671 commit 035c55f

File tree

5 files changed

+302
-18
lines changed

5 files changed

+302
-18
lines changed

‎PluginLists/FlaxProject.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace PluginLists;
2+
3+
public class FlaxProject
4+
{
5+
public string? Name { get; set; }
6+
public string? Version { get; set; }
7+
public string? Company { get; set; }
8+
public string? Copyright { get; set; }
9+
public string? GameTarget { get; set; }
10+
public string? EditorTarget { get; set; }
11+
public string? MinEngineVersion { get; set; }
12+
}

‎PluginLists/PluginDescription.cs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
namespace PluginLists;
2+
3+
public class PluginDescription
4+
{
5+
public string? Name;
6+
public Version? Version;
7+
public string? Author;
8+
public string? AuthorUrl;
9+
public string? HomepageUrl;
10+
public string? RepositoryUrl;
11+
public string? Description;
12+
public string? Category;
13+
public bool IsBeta = false;
14+
public bool IsAlpha = false;
15+
}

‎PluginLists/PluginLists.cs

Lines changed: 208 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,35 @@
11
using System.Diagnostics;
22
using System.Text.Json;
3+
using System.Text.RegularExpressions;
34

45
namespace PluginLists;
56

6-
public class PluginLists
7+
public partial class PluginLists
78
{
9+
[GeneratedRegex(@"(\w+)\s*=\s*(null|false|true|\x22[^\x22]+\x22)", RegexOptions.IgnoreCase, "en-US")]
10+
private static partial Regex VariableAssignmentRegex();
11+
[GeneratedRegex(@"new Version\((\d+),\s*(\d+)\)", RegexOptions.IgnoreCase, "en-US")]
12+
private static partial Regex NewVersionRegex();
13+
[GeneratedRegex(@"(https?://(\w+\.)?github.com)/([^/]+)/([^/]+)/?", RegexOptions.IgnoreCase, "en-US")]
14+
private static partial Regex GithubRepo();
815
private const string LocalListUrl = "pluginlist.json";
916
private readonly HttpClient _httpClient = new();
10-
private readonly ISet<string> _plugins;
17+
private readonly ISet<string> _pluginUrls;
1118
private readonly HashSet<string> _lists;
12-
private int _tasksRunning;
19+
private readonly Dictionary<string, PluginDescription> _plugins;
20+
private readonly Dictionary<string, FlaxProject> _projects;
21+
private int _scannerTasksRunning;
22+
private int _getDetailsTasksRunning;
1323

1424
public PluginLists()
1525
{
16-
_plugins = new HashSet<string>();
26+
_pluginUrls = new HashSet<string>();
1727
_lists = new HashSet<string>();
18-
_tasksRunning = 0;
28+
_plugins = new Dictionary<string, PluginDescription>();
29+
_projects = new Dictionary<string, FlaxProject>();
30+
_scannerTasksRunning = 0;
31+
_getDetailsTasksRunning = 0;
32+
_httpClient.DefaultRequestHeaders.Add("User-Agent", "FlaxPluginList (github.com/nothingTVatYT/FlaxPluginScanner)");
1933
Init();
2034
}
2135

@@ -58,7 +72,7 @@ private void Init()
5872
foreach (var url in localList.Plugins)
5973
{
6074
Debug.WriteLine($"Adding {url}");
61-
lock (_plugins) _plugins.Add(url);
75+
lock (_pluginUrls) _pluginUrls.Add(url);
6276
}
6377

6478
foreach (var url in localList.Lists)
@@ -74,22 +88,26 @@ public async void AddList(string url)
7488
if (_lists.Contains(url))
7589
return;
7690
}
77-
Interlocked.Increment(ref _tasksRunning);
91+
Interlocked.Increment(ref _scannerTasksRunning);
7892
lock (_lists) _lists.Add(url);
7993
var stream = await _httpClient.GetStreamAsync(url);
8094
var list = JsonSerializer.Deserialize<PluginLinkList>(stream);
8195
if (list == null)
8296
{
8397
Debug.WriteLine($"Not a plugin link list: {url}");
84-
Interlocked.Decrement(ref _tasksRunning);
98+
Interlocked.Decrement(ref _scannerTasksRunning);
8599
return;
86100
}
87101

88102
// scan plugin URLs
89103
foreach (var pluginUrl in list.Plugins)
90104
{
91105
Debug.WriteLine($"Adding {pluginUrl}");
92-
lock (_plugins) _plugins.Add(pluginUrl);
106+
lock (_pluginUrls)
107+
{
108+
if (_pluginUrls.Add(pluginUrl))
109+
FindPluginDescription(pluginUrl);
110+
}
93111
}
94112

95113
// scan list links
@@ -99,7 +117,173 @@ public async void AddList(string url)
99117
AddList(listUrl);
100118
}
101119

102-
Interlocked.Decrement(ref _tasksRunning);
120+
Interlocked.Decrement(ref _scannerTasksRunning);
121+
}
122+
123+
private async void FindPluginDescription(string url)
124+
{
125+
var match = GithubRepo().Match(url);
126+
if (!match.Success)
127+
{
128+
Debug.WriteLine($"Not a github URL: {url}");
129+
return;
130+
}
131+
132+
var owner = match.Groups[3].Value;
133+
var repository = match.Groups[4].Value;
134+
if (string.IsNullOrEmpty(owner) || string.IsNullOrEmpty(repository))
135+
{
136+
Debug.WriteLine($"not a github repository URL: {url}");
137+
for (var i = 0; i < match.Groups.Count; i++)
138+
Debug.WriteLine($" group {i} {match.Groups[i].Value}");
139+
return;
140+
}
141+
Interlocked.Increment(ref _getDetailsTasksRunning);
142+
// get default branch
143+
var requestUri = $"https://api.github.com/repos/{owner}/{repository}";
144+
var repositoryDescription = await GetObjectFromUrl<RepositoryDescription>(requestUri);
145+
var branch = repositoryDescription?.default_branch;
146+
if (string.IsNullOrEmpty(branch))
147+
{
148+
Debug.WriteLine($"Could not get default branch for {url}.");
149+
Interlocked.Decrement(ref _getDetailsTasksRunning);
150+
return;
151+
}
152+
153+
// get FlaxProject and Source folder hash
154+
var treesResult =
155+
await GetObjectFromUrl<TreesResult>($"https://api.github.com/repos/{owner}/{repository}/git/trees/{branch}");
156+
var sourceFolderUrl = "";
157+
FlaxProject? flaxProject = null;
158+
if (treesResult is { tree: not null })
159+
{
160+
foreach (var tree in treesResult.tree)
161+
{
162+
if (tree.path != null && tree.path.EndsWith(".flaxproj") && "blob".Equals(tree.type) && tree.url != null)
163+
{
164+
flaxProject = await GetObjectFromBlob<FlaxProject>(tree.url);
165+
if (flaxProject != null)
166+
lock (_projects)
167+
_projects[url] = flaxProject;
168+
}
169+
170+
if ("Source".Equals(tree.path) && "tree".Equals(tree.type))
171+
{
172+
sourceFolderUrl = tree.url;
173+
}
174+
}
175+
}
176+
177+
if (!string.IsNullOrEmpty(sourceFolderUrl) && flaxProject is { Name: not null })
178+
{
179+
var sourceTreesResult = await GetObjectFromUrl<TreesResult>(sourceFolderUrl);
180+
if (sourceTreesResult is { tree: not null })
181+
{
182+
foreach (var sourceFileTree in sourceTreesResult.tree)
183+
{
184+
if (flaxProject.Name.Equals(sourceFileTree.path))
185+
{
186+
// this is the Source folder for the GamePlugin
187+
var gameSourceTreesResult = await GetObjectFromUrl<TreesResult>(sourceFileTree.url);
188+
if (gameSourceTreesResult is { tree: not null })
189+
{
190+
// looking for the GamePlugin constructor
191+
// the name could be anything
192+
foreach (var gameSourceFilesResult in gameSourceTreesResult.tree)
193+
{
194+
if (gameSourceFilesResult.path != null && gameSourceFilesResult.path.EndsWith(".cs"))
195+
{
196+
var text = await GetTextFile(gameSourceFilesResult.url);
197+
if (text.Contains("new PluginDescription"))
198+
{
199+
var pluginDescription = ParsePluginDescription(text);
200+
if (pluginDescription != null)
201+
lock (_plugins) _plugins[url] = pluginDescription;
202+
break;
203+
}
204+
}
205+
}
206+
}
207+
}
208+
}
209+
}
210+
211+
}
212+
213+
Interlocked.Decrement(ref _getDetailsTasksRunning);
214+
}
215+
216+
private async Task<T?> GetObjectFromUrl<T>(string? url)
217+
{
218+
if (url == null)
219+
return default;
220+
try
221+
{
222+
var stream = await _httpClient.GetStreamAsync(url);
223+
var result = JsonSerializer.Deserialize<T>(stream);
224+
if (result == null)
225+
Debug.WriteLine($"Could not get a {typeof(T)} from {url}.");
226+
return result;
227+
}
228+
catch (Exception e)
229+
{
230+
Debug.WriteLine($"Could not access {url}: {e}");
231+
return default;
232+
}
233+
}
234+
235+
private async Task<T?> GetObjectFromBlob<T>(string url)
236+
{
237+
var text = await GetTextFile(url);
238+
if (string.IsNullOrEmpty(text))
239+
return default;
240+
try
241+
{
242+
var result = JsonSerializer.Deserialize<T>(text);
243+
return result;
244+
}
245+
catch (Exception e)
246+
{
247+
File.WriteAllText("/tmp/blob.txt", text);
248+
Debug.WriteLine($"Could not deserialize to a {typeof(T)}: {e}");
249+
Debug.WriteLine($" The json is: {text}");
250+
return default;
251+
}
252+
}
253+
254+
private async Task<string> GetTextFile(string? url)
255+
{
256+
if (url == null)
257+
return "";
258+
try
259+
{
260+
var stream = await _httpClient.GetStreamAsync(url);
261+
var blobResult = JsonSerializer.Deserialize<BlobResult>(stream);
262+
if (blobResult is not { content: not null }) return "";
263+
var text = System.Text.Encoding.ASCII.GetString(Convert.FromBase64String(blobResult.content));
264+
var idx = text.IndexOf('{');
265+
if (idx > 0)
266+
text = text[idx..];
267+
// if (!string.IsNullOrEmpty(text))
268+
// text = text[2..];
269+
return text;
270+
}
271+
catch (Exception e)
272+
{
273+
Debug.WriteLine($"Could not get text file from {url}: {e}");
274+
return "";
275+
}
276+
}
277+
278+
private PluginDescription? ParsePluginDescription(string text)
279+
{
280+
var start = text.IndexOf("new PluginDescription", StringComparison.Ordinal);
281+
var openBracket = text.IndexOf("{", start, StringComparison.Ordinal);
282+
var closeBracket = text.IndexOf("}", openBracket, StringComparison.Ordinal);
283+
var initCode = text.Substring(openBracket, closeBracket - openBracket + 1);
284+
// convert to json
285+
var replaced = VariableAssignmentRegex().Replace(NewVersionRegex().Replace(initCode, @"{ Major: \1, Minor: \2 }"), "\"\\1\" : \\2");
286+
return JsonSerializer.Deserialize<PluginDescription>(replaced);
103287
}
104288

105289
/// <summary>
@@ -108,7 +292,12 @@ public async void AddList(string url)
108292
/// <returns>true if at least one tasks is still active</returns>
109293
public bool IsScanning()
110294
{
111-
return _tasksRunning > 0;
295+
return _scannerTasksRunning > 0;
296+
}
297+
298+
public bool IsGettingDetails()
299+
{
300+
return _getDetailsTasksRunning > 0;
112301
}
113302

114303
/// <summary>
@@ -118,7 +307,14 @@ public bool IsScanning()
118307
public List<string> GetPluginUrls()
119308
{
120309
List<string> result;
121-
lock (_plugins) result = _plugins.ToList();
310+
lock (_pluginUrls) result = _pluginUrls.ToList();
122311
return result;
123312
}
313+
314+
public PluginDescription? GetPluginDescription(string url)
315+
{
316+
PluginDescription? p;
317+
lock (_plugins) _plugins.TryGetValue(url, out p);
318+
return p;
319+
}
124320
}

‎PluginLists/Program.cs

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,16 @@
33
Trace.Listeners.Add(new TextWriterTraceListener(Console.Out));
44

55
var pl = new PluginLists.PluginLists();
6-
while (pl.IsScanning())
6+
while (pl.IsScanning() || pl.IsGettingDetails())
77
{
88
Thread.Sleep(100);
9-
foreach (var url in pl.GetPluginUrls())
10-
{
11-
Console.WriteLine(url);
12-
}
13-
}
9+
}
10+
11+
foreach (var url in pl.GetPluginUrls())
12+
{
13+
Console.WriteLine(url);
14+
var desc = pl.GetPluginDescription(url);
15+
if (desc != null)
16+
Console.WriteLine($"{desc.Name} {desc.Version} {desc.Description}");
17+
}
18+

‎PluginLists/RepositoryDescription.cs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// ReSharper disable InconsistentNaming
2+
// ReSharper disable UnassignedField.Global
3+
4+
using System.Text.Json.Serialization;
5+
6+
namespace PluginLists;
7+
8+
public class RepositoryDescription
9+
{
10+
[JsonInclude]
11+
public string? name;
12+
[JsonInclude]
13+
public string? full_name;
14+
[JsonInclude]
15+
public string? description;
16+
[JsonInclude]
17+
public bool fork;
18+
[JsonInclude]
19+
public string? git_url;
20+
[JsonInclude]
21+
public string? ssh_url;
22+
[JsonInclude]
23+
public string? clone_url;
24+
[JsonInclude]
25+
public string? default_branch;
26+
}
27+
28+
public class TreeResult
29+
{
30+
[JsonInclude]
31+
public string? path;
32+
[JsonInclude]
33+
public string? type;
34+
[JsonInclude]
35+
public string? sha;
36+
[JsonInclude]
37+
public string? url;
38+
}
39+
40+
public class TreesResult
41+
{
42+
[JsonInclude]
43+
public string? sha;
44+
[JsonInclude]
45+
public string? url;
46+
[JsonInclude]
47+
public List<TreeResult>? tree;
48+
}
49+
50+
public class BlobResult
51+
{
52+
[JsonInclude]
53+
public string? content;
54+
[JsonInclude]
55+
public string? encoding;
56+
}

0 commit comments

Comments
 (0)