PxTools · JohannesFinsveen · Jun 4, 2025 · Jun 10, 2025
@@ -43,5 +43,16 @@ public interface IDataSource
         /// <param name="language">codelist with texts for a specific language</param>
         /// <returns></returns>
         Codelist? GetCodelist(string id, string language);
+
+        /// <summary> Intended for the (Lucene) indexing.
+        /// Returns a list of tableid (url-type not datasource-type) for tables where published is in the intervall [from,to]
+        /// For cnmm the tableid is the maintable.tableid column and time of publication is the content.published column.
+        /// Does not restrict the output-list to things like PresCategory = public or table "is running", since
+        /// we want to run it with internal DBs.
+        /// </summary>
+        /// <param name="from">Earliest. MinDate. Inclusive</param>
+        /// <param name="to">Lastest. MaxDate. Inclusive</param>
+        /// <returns>A list of (url-type) tableids (cnmm:maintable.tableid) which may be empty</returns>
+        List<string> GetTablesPublishedBetween(DateTime from, DateTime to);
     }
 }
@@ -1,5 +1,7 @@
 
 
+using System;
+
 namespace PxWeb.UnitTests.DataSource
 {
     [TestClass]
@@ -224,6 +226,37 @@ public void ShouldNotResolveTablePath()
             Assert.IsFalse(selectionExists);
         }
 
+        [TestMethod]
+        public void GetTablesPublishedBetween_WhenCalled_ReturnNonNull()
+        {
+            string pathRunning = Directory.GetCurrentDirectory();
+            int index = pathRunning.IndexOf("PxWeb.UnitTests");
+
+            if (index == -1)
+            {
+                throw new System.Exception("Hmm, Directory.GetCurrentDirectory() does not contain string:PxWeb.UnitTests , so unable to find wwwroot path.");
+            }
+            string repoRoot = pathRunning.Substring(0, index);
+            string wwwPath = Path.Combine(repoRoot, "PxWeb", "wwwroot");
+
+            var hostingEnvironmentMock = new Mock<IPxHost>();
+            hostingEnvironmentMock
+                .Setup(m => m.RootPath)
+                .Returns(wwwPath);
+
+            var dataSource = new PxFileDataSource(
+                new Mock<IPxFileConfigurationService>().Object,
+                new Mock<IItemSelectionResolver>().Object,
+                new Mock<ITablePathResolver>().Object,
+                hostingEnvironmentMock.Object,
+                new Mock<ICodelistMapper>().Object);
+
+            var updataedTables = dataSource.GetTablesPublishedBetween(new DateTime(2023, 8, 1), new DateTime(2023, 9, 1));
+
+            Assert.IsNotNull(updataedTables);
+
+        }
+
         private TablePathResolverPxFile GetTablePathResolver()
         {
             var testFactory = new TestFactory();
@@ -260,5 +293,7 @@ private TablePathResolverPxFile GetTablePathResolver()
 
             return resolver;
         }
+
+
     }
 }
@@ -116,7 +116,7 @@
                                item.SortCode = item.Text;
                            }
                        };
                        m.Restriction = item => { return true; }; // TODO: Will show all tables! Even though they are not published...
                    });
            retMenu.RootItem.Sort();

@@ -249,5 +249,10 @@
 
             return codelist;
         }
+
+        public List<string> GetTablesPublishedBetween(DateTime from, DateTime to)
+        {
+            return PCAxis.Sql.ApiUtils.ApiUtilStatic.GetTablesPublishedBetween(from, to);
+        }
     }
 }
@@ -15,7 +15,7 @@
 {
    public class PxFileDataSource : IDataSource
    {
        private readonly IPxFileConfigurationService _pxFileConfigurationService;
        private readonly IItemSelectionResolver _itemSelectionResolver;
        private readonly ITablePathResolver _tablePathResolver;
        private readonly IPxHost _hostingEnvironment;
@@ -103,7 +103,7 @@
            menu.SetCurrentItemBySelection(itmSel.Menu, itmSel.Selection);

            // Fix selection for subitems - we only want the last part...
            if (menu.CurrentItem is PxMenuItem)
            {
                foreach (var item in ((PxMenuItem)(menu.CurrentItem)).SubItems)
                {
@@ -154,7 +154,7 @@

        }

        private string GetIdentifierWithoutPath(string id)
        {
            if (id.Contains('/'))
            {
@@ -166,5 +166,40 @@
             }
         }
 
+        public List<string> GetTablesPublishedBetween(DateTime from, DateTime to)
+        {
+            MenuXmlFile menuXmlFile = new MenuXmlFile(_hostingEnvironment);
+            var doc = menuXmlFile.GetAsXmlDocument();
+
+            var tableIds = new List<string>();
+            var nodes = doc.SelectNodes("//Link[LastUpdated]");
+
+            if (nodes is null)
+            {
+                return tableIds;
+            }
+
+            foreach (XmlNode link in nodes)
+            {
+                var lastUpdatedNode = link.SelectSingleNode("LastUpdated");
+                if (lastUpdatedNode != null)
+                {
+                    if (DateTime.TryParse(lastUpdatedNode.InnerText, out DateTime lastUpdated))
+                    {
+                        if (lastUpdated >= from && lastUpdated <= to)
+                        {
+                            string? tableId = link.Attributes?["tableId"]?.Value;
+
+                            if (tableId is not null && !tableIds.Contains(tableId))
+                            {
+                                tableIds.Add(tableId);
+                            }
+                        }
+                    }
+                }
+            }
+
+            return tableIds;
+        }
     }
 }
@@ -1,6 +1,7 @@
 using System.ComponentModel.DataAnnotations;
 using System.Linq;
 using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
 
 using Microsoft.AspNetCore.Mvc;
@@ -17,7 +18,7 @@
 {
    [ApiController]
    //[ApiExplorerSettings(IgnoreApi = true)]
    public class SearchindexController : ControllerBase
    {
        private readonly IDataSource _dataSource;
        private readonly ISearchBackend _backend;
@@ -47,32 +48,53 @@
         [SwaggerOperation("IndexDatabase")]
         [SwaggerResponse(statusCode: 202, description: "Accepted")]
         [SwaggerResponse(statusCode: 401, description: "Unauthorized")]
-        public IActionResult IndexDatabase()
+        public IActionResult IndexDatabase(int? pastHours)
         {
             _backgroundWorkerQueue.QueueBackgroundWorkItem(async token =>
             {
-                try
+                if (pastHours is not null)
                 {
-                    List<string> languages = new List<string>();
+                    try
+                    {
+                        DateTime to = DateTime.Now;
+                        DateTime from = to - TimeSpan.FromHours(pastHours.Value);
+                        List<string> tableList = _dataSource.GetTablesPublishedBetween(from, to);
 
-                    var config = _pxApiConfigurationService.GetConfiguration();
+                        string message = $"Looked for tables published between {from:yyyy-MM-dd HH:mm:ss} and {to:yyyy-MM-dd HH:mm:ss}. Found {tableList.Count()}";
 
-                    if (config.Languages.Count == 0)
-                    {
-                        _logger.LogError("No languages configured for PxApi. New index will not be created.");
-                        return;
+                        _responseState.AddEvent(new Event("Information", message));
+                        _logger.LogDebug(message);
+                        if (tableList.Count > 0)
+                        {
+
+                            await UpdateFromTableList(tableList, token);
+                        }
                     }
-                    foreach (var lang in config.Languages)
+                    catch (System.Exception ex)
                     {
-                        languages.Add(lang.Id);
+                        _responseState.AddEvent(new Event("Error", ex.Message));
+                        _logger.LogError(ex, "Error when building search index");
                     }
-
-                    Indexer indexer = new Indexer(_dataSource, _backend, _logger);
-                    await Task.Run(() => indexer.IndexDatabase(languages), token);
                 }
-                catch (System.Exception ex)
+                else
                 {
-                    _logger.LogError(ex, "Error when building search index");
+                    try
+                    {
+                        //TODO: this factoring adds _responseState.AddEvent(new Event("Error", message));   Good thing, likp?
+                        List<string> languages = GetLangaugesFromConfig();
+                        if (languages.Count == 0)
+                        {
+                            return;
+                        }
+
+ 
10000
                       Indexer indexer = new Indexer(_dataSource, _backend, _logger);
+                        await Task.Run(() => indexer.IndexDatabase(languages), token);
+                    }
+                    catch (System.Exception ex)
+                    {
+                        _responseState.AddEvent(new Event("Error", ex.Message));
+                        _logger.LogError(ex, "Error when building search index");
+                    }
                 }
             });
             return new AcceptedResult();
@@ -94,46 +116,43 @@
             {
                 try
                 {
-                    List<string> languages = new List<string>();
                     List<string> tableList = tables
                         .Select(table => Regex.Replace(table.Trim(), @"[^0-9a-zA-Z]+", "", RegexOptions.None, TimeSpan.FromMilliseconds(100)))
                         .ToList();
 
-                    if (tableList.Count == 0)
-                    {
-                        string message = "Incoming list with table id's to be updated is empty. Index will not be updated.";
-                        _logger.LogError(message);
-                        _responseState.AddEvent(new Event("Error", message));
-                        return;
-                    }
-
-                    var config = _pxApiConfigurationService.GetConfiguration();
-
-                    if (config.Languages.Count == 0)
-                    {
-                        string message = "No languages configured for PxApi. Index will not be updated.";
-                        _logger.LogError(message);
-                        _responseState.AddEvent(new Event("Error", message));
-                        return;
-                    }
-
-                    foreach (var lang in config.Languages)
-                    {
-                        languages.Add(lang.Id);
-                    }
-
-                    Indexer indexer = new Indexer(_dataSource, _backend, _logger);
-                    await Task.Run(() => indexer.UpdateTableEntries(tableList, languages), token);
+                    await UpdateFromTableList(tableList, token);
                 }
                 catch (System.Exception ex)
                 {
                     _responseState.AddEvent(new Event("Error", ex.Message));
-                    _logger.LogError(ex.Message);
+                    _logger.LogError(ex, ex.Message);
                 }
+
+
             });
             return new AcceptedResult();
         }
 
+        private async Task UpdateFromTableList(List<string> tableList, CancellationToken token)
+        {
+            if (tableList.Count == 0)
+            {
+                string message = "Incoming list with table id's to be updated is empty. Index will not be updated.";
+                _logger.LogError(message);
+                _responseState.AddEvent(new Event("Error", message));
+                return;
+            }
+
+            List<string> languages = GetLangaugesFromConfig();
+            if (languages.Count == 0)
+            {
+                return;
+            }
+
+            Indexer indexer = new Indexer(_dataSource, _backend, _logger);
+            await Task.Run(() => indexer.UpdateTableEntries(tableList, languages), token);
+        }
+
         [HttpGet]
         [Route("/admin/searchindex")]
         [SwaggerOperation("IndexDatabase")]
@@ -143,5 +162,25 @@
         {
             return new JsonResult(_responseState.Data);
         }
+
+        private List<string> GetLangaugesFromConfig()
+        {
+            List<string> languages = new List<string>();
+            var config = _pxApiConfigurationService.GetConfiguration();
+
+            if (config.Languages.Count == 0)
+            {
+                string message = "No languages configured for PxApi. Index will not be updated.";
+                _logger.LogError(message);
+                _responseState.AddEvent(new Event("Error", message));
+                return languages;
+            }
+
+            foreach (var lang in config.Languages)
+            {
+                languages.Add(lang.Id);
+            }
+            return languages;
+        }
     }
 }
@@ -48,7 +48,7 @@
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="PCAxis.Menu.ConfigDatamodelMenu" Version="1.0.8" />
     <PackageReference Include="PCAxis.Serializers" Version="1.9.0" />
-    <PackageReference Include="PcAxis.Sql" Version="1.4.1" />
+    <PackageReference Include="PcAxis.Sql" Version="1.4.3" />
     <PackageReference Include="PxWeb.Api2.Server" Version="2.0.0-beta.17" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="8.1.1" />
     <PackageReference Include="Swashbuckle.AspNetCore.Annotations" Version="8.1.1" />