From 19fafd85c01fac961485382faa9351ba84b99015 Mon Sep 17 00:00:00 2001 From: Johannes Date: Wed, 4 Jun 2025 13:21:16 +0200 Subject: [PATCH 1/2] New Feature in Admin/Searchindex past hour parameter Bumping sql to 1.4.3 --- Px.Abstractions/Interfaces/IDataSource.cs | 11 ++ .../Api2/DataSource/Cnmm/CnmmDataSource.cs | 5 + .../DataSource/PxFile/PxFileDataSource.cs | 4 + .../Api2/Admin/SearchindexController.cs | 123 ++++++++++++------ PxWeb/PxWeb.csproj | 2 +- 5 files changed, 102 insertions(+), 43 deletions(-) diff --git a/Px.Abstractions/Interfaces/IDataSource.cs b/Px.Abstractions/Interfaces/IDataSource.cs index 661ea03f..dedc82f3 100644 --- a/Px.Abstractions/Interfaces/IDataSource.cs +++ b/Px.Abstractions/Interfaces/IDataSource.cs @@ -43,5 +43,16 @@ public interface IDataSource /// codelist with texts for a specific language /// Codelist? GetCodelist(string id, string language); + + /// Intended for the (Lucene) indexing. + /// Returns a list of tableid (url-type not datasource-type) for tables where published is in the intervall [from,to] + /// For cnmm the tableid is the maintable.tableid column and time of publication is the content.published column. + /// Does not restrict the output-list to things like PresCategory = public or table "is running", since + /// we want to run it with internal DBs. + /// + /// Earliest. MinDate. Inclusive + /// Lastest. MaxDate. Inclusive + /// A list of (url-type) tableids (cnmm:maintable.tableid) which may be empty + List GetTablesPublishedBetween(DateTime from, DateTime to); } } diff --git a/PxWeb/Code/Api2/DataSource/Cnmm/CnmmDataSource.cs b/PxWeb/Code/Api2/DataSource/Cnmm/CnmmDataSource.cs index 25d6ea92..b0808784 100644 --- a/PxWeb/Code/Api2/DataSource/Cnmm/CnmmDataSource.cs +++ b/PxWeb/Code/Api2/DataSource/Cnmm/CnmmDataSource.cs @@ -249,5 +249,10 @@ private static bool IsInteger(string value) return codelist; } + + public List GetTablesPublishedBetween(DateTime from, DateTime to) + { + return PCAxis.Sql.ApiUtils.ApiUtilStatic.GetTablesPublishedBetween(from, to); + } } } diff --git a/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs b/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs index 52dae0c5..429c01ad 100644 --- a/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs +++ b/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs @@ -166,5 +166,9 @@ private string GetIdentifierWithoutPath(string id) } } + public List GetTablesPublishedBetween(DateTime from, DateTime to) + { + throw new NotImplementedException(); + } } } diff --git a/PxWeb/Controllers/Api2/Admin/SearchindexController.cs b/PxWeb/Controllers/Api2/Admin/SearchindexController.cs index a363aa54..4010728a 100644 --- a/PxWeb/Controllers/Api2/Admin/SearchindexController.cs +++ b/PxWeb/Controllers/Api2/Admin/SearchindexController.cs @@ -1,6 +1,7 @@ using System.ComponentModel.DataAnnotations; using System.Linq; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; @@ -47,32 +48,53 @@ public SearchindexController(BackgroundWorkerQueue backgroundWorkerQueue, IContr [SwaggerOperation("IndexDatabase")] [SwaggerResponse(statusCode: 202, description: "Accepted")] [SwaggerResponse(statusCode: 401, description: "Unauthorized")] - public IActionResult IndexDatabase() + public IActionResult IndexDatabase(int? pastHours) { _backgroundWorkerQueue.QueueBackgroundWorkItem(async token => { - try + if (pastHours is not null) { - List languages = new List(); + try + { + DateTime to = DateTime.Now; + DateTime from = to - TimeSpan.FromHours(pastHours.Value); + List tableList = _dataSource.GetTablesPublishedBetween(from, to); - var config = _pxApiConfigurationService.GetConfiguration(); + string message = $"Looked for tables published between {from:yyyy-MM-dd HH:mm:ss} and {to:yyyy-MM-dd HH:mm:ss}. Found {tableList.Count()}"; - if (config.Languages.Count == 0) - { - _logger.LogError("No languages configured for PxApi. New index will not be created."); - return; + _responseState.AddEvent(new Event("Information", message)); + _logger.LogDebug(message); + if (tableList.Count > 0) + { + + await UpdateFromTableList(tableList, token); + } } - foreach (var lang in config.Languages) + catch (System.Exception ex) { - languages.Add(lang.Id); + _responseState.AddEvent(new Event("Error", ex.Message)); + _logger.LogError(ex, "Error when building search index"); } - - Indexer indexer = new Indexer(_dataSource, _backend, _logger); - await Task.Run(() => indexer.IndexDatabase(languages), token); } - catch (System.Exception ex) + else { - _logger.LogError(ex, "Error when building search index"); + try + { + //TODO: this factoring adds _responseState.AddEvent(new Event("Error", message)); Good thing, likp? + List languages = GetLangaugesFromConfig(); + if (languages.Count == 0) + { + return; + } + + Indexer indexer = new Indexer(_dataSource, _backend, _logger); + await Task.Run(() => indexer.IndexDatabase(languages), token); + } + catch (System.Exception ex) + { + _responseState.AddEvent(new Event("Error", ex.Message)); + _logger.LogError(ex, "Error when building search index"); + } } }); return new AcceptedResult(); @@ -94,46 +116,43 @@ public IActionResult IndexDatabase([FromBody, Required] string[] tables) { try { - List languages = new List(); List tableList = tables .Select(table => Regex.Replace(table.Trim(), @"[^0-9a-zA-Z]+", "", RegexOptions.None, TimeSpan.FromMilliseconds(100))) .ToList(); - if (tableList.Count == 0) - { - string message = "Incoming list with table id's to be updated is empty. Index will not be updated."; - _logger.LogError(message); - _responseState.AddEvent(new Event("Error", message)); - return; - } - - var config = _pxApiConfigurationService.GetConfiguration(); - - if (config.Languages.Count == 0) - { - string message = "No languages configured for PxApi. Index will not be updated."; - _logger.LogError(message); - _responseState.AddEvent(new Event("Error", message)); - return; - } - - foreach (var lang in config.Languages) - { - languages.Add(lang.Id); - } - - Indexer indexer = new Indexer(_dataSource, _backend, _logger); - await Task.Run(() => indexer.UpdateTableEntries(tableList, languages), token); + await UpdateFromTableList(tableList, token); } catch (System.Exception ex) { _responseState.AddEvent(new Event("Error", ex.Message)); - _logger.LogError(ex.Message); + _logger.LogError(ex, ex.Message); } + + }); return new AcceptedResult(); } + private async Task UpdateFromTableList(List tableList, CancellationToken token) + { + if (tableList.Count == 0) + { + string message = "Incoming list with table id's to be updated is empty. Index will not be updated."; + _logger.LogError(message); + _responseState.AddEvent(new Event("Error", message)); + return; + } + + List languages = GetLangaugesFromConfig(); + if (languages.Count == 0) + { + return; + } + + Indexer indexer = new Indexer(_dataSource, _backend, _logger); + await Task.Run(() => indexer.UpdateTableEntries(tableList, languages), token); + } + [HttpGet] [Route("/admin/searchindex")] [SwaggerOperation("IndexDatabase")] @@ -143,5 +162,25 @@ public IActionResult GetState() { return new JsonResult(_responseState.Data); } + + private List GetLangaugesFromConfig() + { + List languages = new List(); + var config = _pxApiConfigurationService.GetConfiguration(); + + if (config.Languages.Count == 0) + { + string message = "No languages configured for PxApi. Index will not be updated."; + _logger.LogError(message); + _responseState.AddEvent(new Event("Error", message)); + return languages; + } + + foreach (var lang in config.Languages) + { + languages.Add(lang.Id); + } + return languages; + } } } diff --git a/PxWeb/PxWeb.csproj b/PxWeb/PxWeb.csproj index 569e71e4..796331bf 100644 --- a/PxWeb/PxWeb.csproj +++ b/PxWeb/PxWeb.csproj @@ -48,7 +48,7 @@ - + From 8109cf8d46770bcd8b6efadbf9d5f9622b09834d Mon Sep 17 00:00:00 2001 From: likp Date: Tue, 10 Jun 2025 19:52:18 +0200 Subject: [PATCH 2/2] Added `PxFileDataSource` implementation for `GetTablesPublishedBetween` --- .../DataSource/PXDataSourceTest.cs | 35 +++++++++++++++++++ .../DataSource/PxFile/PxFileDataSource.cs | 33 ++++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/PxWeb.UnitTests/DataSource/PXDataSourceTest.cs b/PxWeb.UnitTests/DataSource/PXDataSourceTest.cs index 54e753ea..ddb8cd06 100644 --- a/PxWeb.UnitTests/DataSource/PXDataSourceTest.cs +++ b/PxWeb.UnitTests/DataSource/PXDataSourceTest.cs @@ -1,5 +1,7 @@  +using System; + namespace PxWeb.UnitTests.DataSource { [TestClass] @@ -224,6 +226,37 @@ public void ShouldNotResolveTablePath() Assert.IsFalse(selectionExists); } + [TestMethod] + public void GetTablesPublishedBetween_WhenCalled_ReturnNonNull() + { + string pathRunning = Directory.GetCurrentDirectory(); + int index = pathRunning.IndexOf("PxWeb.UnitTests"); + + if (index == -1) + { + throw new System.Exception("Hmm, Directory.GetCurrentDirectory() does not contain string:PxWeb.UnitTests , so unable to find wwwroot path."); + } + string repoRoot = pathRunning.Substring(0, index); + string wwwPath = Path.Combine(repoRoot, "PxWeb", "wwwroot"); + + var hostingEnvironmentMock = new Mock(); + hostingEnvironmentMock + .Setup(m => m.RootPath) + .Returns(wwwPath); + + var dataSource = new PxFileDataSource( + new Mock().Object, + new Mock().Object, + new Mock().Object, + hostingEnvironmentMock.Object, + new Mock().Object); + + var updataedTables = dataSource.GetTablesPublishedBetween(new DateTime(2023, 8, 1), new DateTime(2023, 9, 1)); + + Assert.IsNotNull(updataedTables); + + } + private TablePathResolverPxFile GetTablePathResolver() { var testFactory = new TestFactory(); @@ -260,5 +293,7 @@ private TablePathResolverPxFile GetTablePathResolver() return resolver; } + + } } diff --git a/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs b/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs index 429c01ad..3f210f9c 100644 --- a/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs +++ b/PxWeb/Code/Api2/DataSource/PxFile/PxFileDataSource.cs @@ -168,7 +168,38 @@ private string GetIdentifierWithoutPath(string id) public List GetTablesPublishedBetween(DateTime from, DateTime to) { - throw new NotImplementedException(); + MenuXmlFile menuXmlFile = new MenuXmlFile(_hostingEnvironment); + var doc = menuXmlFile.GetAsXmlDocument(); + + var tableIds = new List(); + var nodes = doc.SelectNodes("//Link[LastUpdated]"); + + if (nodes is null) + { + return tableIds; + } + + foreach (XmlNode link in nodes) + { + var lastUpdatedNode = link.SelectSingleNode("LastUpdated"); + if (lastUpdatedNode != null) + { + if (DateTime.TryParse(lastUpdatedNode.InnerText, out DateTime lastUpdated)) + { + if (lastUpdated >= from && lastUpdated <= to) + { + string? tableId = link.Attributes?["tableId"]?.Value; + + if (tableId is not null && !tableIds.Contains(tableId)) + { + tableIds.Add(tableId); + } + } + } + } + } + + return tableIds; } } }