vividus-framework · valfirst · Nov 29, 2023 · Nov 28, 2023
@@ -182,6 +182,16 @@ a|`true`
 |`true`
 |Whether to crawl https pages.
 
+|`transformer.from-headless-crawling.http.headers.<header name>=<header value>`
+|
+|
+a|Set of headers to set for every crawling request being sent.
+
+[source,properties]
+----
+transformer.from-headless-crawling.http.headers.x-vercel-protection-bypass=1fac2b25014d35e5103b
+----
+
 4+^.^|_Proxy_
 
 |`transformer.from-headless-crawling.proxy-host`

@@ -19,11 +19,15 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
+import java.util.Collection;
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.hc.core5.http.Header;
 import org.vividus.util.UriUtils;
 import org.vividus.util.UriUtils.UserInfo;
 import org.vividus.util.property.IPropertyMapper;
@@ -97,7 +101,23 @@ private CrawlConfig createCrawlConfig(URI mainApplicationPage) throws IOExceptio
     }
 
     @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonAutoDetect(fieldVisibility = Visibility.NONE)
+    @SuppressWarnings("unused")
     private static final class MappedCrawlConfig extends CrawlConfig
     {
+        private Http http;
+
+        private void setHttp(Http http)
+        {
+            this.http = http;
+        }
+
+        public final class Http
+        {
+            private void setHeaders(Collection<? extends Header> headers)
+            {
+                setDefaultHeaders(headers);
+            }
+        }
     }
 }
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.vividus.databind;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import org.apache.hc.core5.http.message.BasicHeader;
+
+public class BasicHeadersDeserializer extends JsonDeserializer<Collection<BasicHeader>>
+{
+    @Override
+    public Collection<BasicHeader> deserialize(JsonParser parser, DeserializationContext context) throws IOException
+    {
+        JsonNode node = parser.getCodec().readTree(parser);
+        Collection<BasicHeader> headers = new ArrayList<>();
+        node.fields().forEachRemaining(f ->
+        {
+            String headerName = f.getKey();
+            String headerValue = f.getValue().asText();
+            headers.add(new BasicHeader(headerName, headerValue));
+        });
+        return headers;
+    }
+}
@@ -91,6 +91,8 @@
         <property name="excludeExtensionsRegex" value="${transformer.from-headless-crawling.exclude-extensions-regex}" />
     </bean>
 
+    <bean class="org.vividus.databind.BasicHeadersDeserializer" />
+
     <util:list id="stepBeanNames-WebAppToRestApi" value-type="java.lang.String">
         <idref bean="httpRequestSteps" />
         <idref bean="resourceCheckSteps" />

@@ -34,6 +34,7 @@
 import org.apache.hc.core5.net.URIBuilder;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.vividus.databind.BasicHeadersDeserializer;
 import org.vividus.util.property.IPropertyMapper;
 import org.vividus.util.property.PropertyMapper;
 
@@ -45,7 +46,7 @@ class CrawlControllerFactoryTests
     private static final String URL = "https://example.com";
 
     private final IPropertyMapper propertyMapper = new PropertyMapper(".", PropertyNamingStrategies.KEBAB_CASE, null,
-            Set.of());
+            Set.of(new BasicHeadersDeserializer()));
 
     @Test
     void shouldCreateCrawlController(@TempDir Path baseDirectory) throws URISyntaxException
@@ -55,9 +56,11 @@ void shouldCreateCrawlController(@TempDir Path baseDirectory) throws URISyntaxEx
             var crawlStorage = baseDirectory.resolve(CRAWL_STORAGE_FOLDER_KEY);
             Integer socketTimeout = 10_000;
 
+            String value = "714d9dac8b334a3e0577";
             var config = Map.of(
                 "socket-timeout", socketTimeout.toString(),
-                CRAWL_STORAGE_FOLDER_KEY, crawlStorage.toString()
+                CRAWL_STORAGE_FOLDER_KEY, crawlStorage.toString(),
+                "http.headers.credit-card-pin", value
             );
 
             var factory = new CrawlControllerFactory(config, propertyMapper);
@@ -79,6 +82,11 @@ void shouldCreateCrawlController(@TempDir Path baseDirectory) throws URISyntaxEx
             var authInfo = authInfos.get(0);
             assertEquals(username, authInfo.getUsername());
             assertEquals(password, authInfo.getPassword());
+            var headers = crawlConfig.getDefaultHeaders();
+            assertThat(headers, hasSize(1));
+            var header = headers.iterator().next();
+            assertEquals("credit-card-pin", header.getName());
+            assertEquals(value, header.getValue());
         }
     }
 

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2019-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unle
A04C
ss required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.vividus.databind;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.junit.jupiter.api.Test;
+
+class BasicHeadersDeserializerTests
+{
+    @Test
+    void shouldDeserializerHeaders() throws IOException
+    {
+        String json = """
+            {
+                "header1": "value1",
+                "header2": "value2"
+            }
+            """;
+
+        JsonParser parser  = new JsonFactory().createParser(json);
+        parser.setCodec(new ObjectMapper());
+
+        Collection<BasicHeader> headers = new BasicHeadersDeserializer().deserialize(parser, null);
+        assertThat(headers, hasSize(2));
+        BasicHeader header1 = (BasicHeader) headers.toArray()[0];
+        assertEquals("header1", header1.getName());
+        assertEquals("value1", header1.getValue());
+        BasicHeader header2 = (BasicHeader) headers.toArray()[1];
+        assertEquals("header2", header2.getName());
+        assertEquals("value2", header2.getValue());
+    }
+}