Skip to content

Commit f2e87db

Browse files
valery1707valery1707
committed
LinkChecker: Use OkHttpClient instead of JVM HttpClient
This library supports caching
1 parent 688129e commit f2e87db

File tree

5 files changed

+66
-124
lines changed

5 files changed

+66
-124
lines changed

‎build.gradle.kts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,15 @@ dependencies {
2828
implementation(libs.jackson.databind)
2929
implementation(libs.mockneat)
3030
implementation(libs.bundles.logger.api)
31+
implementation(libs.okhttp.core)
3132

3233
runtimeOnly(libs.logger.impl)
3334

3435
testImplementation(kotlin("test"))
3536
testImplementation(platform(libs.junit))
3637
testImplementation("org.junit.jupiter:junit-jupiter-params")
3738
testImplementation(libs.assertj)
39+
testImplementation(libs.okhttp.mock)
3840

3941
jmhAnnotationProcessor(libs.jmh.ann)
4042
}

‎gradle/libs.versions.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ jmh-ann = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "
1414
logger-api4j = { module = "org.slf4j:slf4j-api", version = "2.0.6" }
1515
logger-api4k = { module = "io.github.microutils:kotlin-logging-jvm", version = "3.0.4" }
1616
logger-impl = { module = "ch.qos.logback:logback-classic", version = "1.4.5" }
17+
okhttp-core = { module = "com.squareup.okhttp3:okhttp", version = "4.10.0" }
18+
okhttp-mock = { module = "com.github.gmazzo:okhttp-mock", version = "1.5.0" }
1719

1820
[bundles]
1921
logger-api = ["logger.api4j", "logger.api4k"]

‎src/main/kotlin/name/valery1707/problem/LinkChecker.kt‎

Lines changed: 26 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
package name.valery1707.problem
22

3+
import okhttp3.Headers
4+
import okhttp3.OkHttpClient
5+
import okhttp3.Request
36
import java.net.URI
4-
import java.net.http.HttpClient
5-
import java.net.http.HttpHeaders
6-
import java.net.http.HttpRequest
7-
import java.net.http.HttpResponse
87
import java.nio.file.Path
98
import java.time.Duration
109
import java.time.Instant
@@ -24,7 +23,7 @@ class LinkChecker(private val root: Path) {
2423
* Сканируем все файлы из директории, ищем в тексте ссылки, проверяем их на доступность
2524
*/
2625
@OptIn(ExperimentalPathApi::class)
27-
fun findInvalid(client: HttpClient): Map<String, String> {
26+
fun findInvalid(client: OkHttpClient): Map<String, String> {
2827
val filePos2uriCheck = root
2928
.walk(PathWalkOption.FOLLOW_LINKS)
3029
.map { root.relativize(it) }
@@ -104,27 +103,28 @@ class LinkChecker(private val root: Path) {
104103
null
105104
}
106105

107-
private fun URI.check(client: HttpClient): Pair<Int, URI> {
108-
val request = HttpRequest.newBuilder(this).GET().build()
106+
private fun URI.check(client: OkHttpClient): Pair<Int, URI> {
107+
val request = Request.Builder().url(this.toURL()).get().build()
109108
// todo Cache
110109
return try {
111110
logger.info("Check: $this")
112-
val response = client.send(request, HttpResponse.BodyHandlers.discarding())
113-
when (response.statusCode()) {
114-
//Redirects: extract new location
115-
in HTTP_REDIRECT -> response.statusCode() to response.headers().firstValue("Location")!!.get().toURI()!!
116-
117-
//Rate limiting: wait and retry
118-
in HTTP_RATE_LIMIT -> {
119-
val now = Instant.now()
120-
val await = response.headers().rateLimitAwait(now) ?: 500
121-
122-
logger.debug("Await: $await ms")
123-
Thread.sleep(await)
124-
check(client)
111+
client.newCall(request).execute().use { response ->
112+
when (response.code) {
113+
//Redirects: extract new location
114+
in HTTP_REDIRECT -> response.code to response.header("Location")!!.toURI()!!
115+
116+
//Rate limiting: wait and retry
117+
in HTTP_RATE_LIMIT -> {
118+
val now = Instant.now()
119+
val await = response.headers.rateLimitAwait(now) ?: 500
120+
121+
logger.debug("Await: $await ms")
122+
Thread.sleep(await)
123+
check(client)
124+
}
125+
126+
else -> response.code to response.request.url.toUri()
125127
}
126-
127-
else -> response.statusCode() to response.uri()
128128
}
129129
} catch (e: Exception) {
130130
logger.error(e) { "Handle error on checking $this" }
@@ -135,13 +135,10 @@ class LinkChecker(private val root: Path) {
135135
private val HTTP_REDIRECT = setOf(301, 302, 307, 308)
136136
private val HTTP_RATE_LIMIT = setOf(403, 429)
137137

138-
private fun HttpHeaders.rateLimitAwait(now: Instant): Long? {
139-
val map = map()
140-
return HTTP_RATE_LIMIT_EXTRACTORS
141-
.flatMap { map[it.key]?.asSequence()?.map { v -> it.value(v.trim(), now) } ?: emptySequence() }
142-
.filterNotNull()
143-
.firstOrNull { it >= 0 }
144-
}
138+
private fun Headers.rateLimitAwait(now: Instant): Long? = HTTP_RATE_LIMIT_EXTRACTORS
139+
.flatMap { values(it.key).asSequence().map { v -> it.value(v.trim(), now) } }
140+
.filterNotNull()
141+
.firstOrNull { it >= 0 }
145142

146143
private val HTTP_RATE_LIMIT_EXTRACTORS: Map<String, (String, Instant) -> Long?> = mapOf(
147144
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#checking-your-rate-limit-status

‎src/test/kotlin/name/valery1707/problem/LinkCheckerTest.kt‎

Lines changed: 34 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,24 @@ package name.valery1707.problem
22

33
import name.valery1707.problem.LinkChecker.Companion.HTTP_DATE_FORMAT
44
import name.valery1707.problem.LinkChecker.Companion.toURI
5+
import okhttp3.Headers.Companion.toHeaders
6+
import okhttp3.OkHttpClient
7+
import okhttp3.mock.MockInterceptor
8+
import okhttp3.mock.body
59
import org.assertj.core.api.Assertions.assertThat
610
import org.assertj.core.api.Assertions.fail
711
import org.assertj.core.api.Assumptions.assumeThat
812
import org.junit.jupiter.api.Test
913
import org.junit.jupiter.params.ParameterizedTest
1014
import org.junit.jupiter.params.provider.ValueSource
11-
import java.net.Authenticator
12-
import java.net.CookieHandler
1315
import java.net.InetSocketAddress
1416
import java.net.ProxySelector
15-
import java.net.URI
16-
import java.net.http.HttpClient
17-
import java.net.http.HttpHeaders
18-
import java.net.http.HttpRequest
19-
import java.net.http.HttpResponse
2017
import java.nio.file.Path
21-
import java.time.Duration
2218
import java.time.Instant
2319
import java.time.ZoneId
2420
import java.util.*
25-
import java.util.concurrent.CompletableFuture
26-
import java.util.concurrent.Executor
27-
import javax.net.ssl.SSLContext
28-
import javax.net.ssl.SSLParameters
29-
import javax.net.ssl.SSLSession
3021
import kotlin.io.path.toPath
3122

32-
typealias ResponseBuilder<T> = (HttpRequest) -> HttpResponse<T>
3323
typealias ResponseMeta = () -> Pair<Int, Map<String, String>>
3424

3525
internal class LinkCheckerTest {
@@ -42,10 +32,9 @@ internal class LinkCheckerTest {
4232
)
4333
internal fun checkReal(path: Path) {
4434
assumeThat(path).isDirectory.isReadable
45-
val client = HttpClient
46-
.newBuilder()
47-
.followRedirects(HttpClient.Redirect.NEVER)
48-
.proxy(proxy)
35+
val client = OkHttpClient.Builder()
36+
.followRedirects(false).followSslRedirects(false)
37+
.proxySelector(proxy)
4938
.build()
5039
val checker = LinkChecker(path)
5140
assertThat(checker.findInvalid(client)).isEmpty()
@@ -67,18 +56,18 @@ internal class LinkCheckerTest {
6756
}
6857

6958
//Check links via: curl --silent -X GET --head 'URL'
70-
val client = MockedHttpClient.fromMeta(
59+
val client = mockHttpClient(
7160
mapOf(
72-
"https://ya.ru" to listOf(
73-
redirect(302, "https://ya.ru/"),
61+
"https://habr.com/ru/company/otus/blog/707724/comments" to mutableListOf(
62+
redirect(302, "https://habr.com/ru/company/otus/blog/707724/comments/"),
7463
),
75-
"https://ya.ru/" to listOf(
64+
"https://habr.com/ru/company/otus/blog/707724/comments/" to mutableListOf(
7665
ok(),
7766
),
78-
"http://schema.org" to listOf(
67+
"http://schema.org/" to mutableListOf(
7968
redirect(301, "https://schema.org/"),
8069
),
81-
"https://github.com/androidx/androidx/blob/androidx-main/build.gradle" to listOf(
70+
"https://github.com/androidx/androidx/blob/androidx-main/build.gradle" to mutableListOf(
8271
//Will wait some time
8372
rateLimitGH(2111),
8473
//Will wait zero time
@@ -87,14 +76,14 @@ internal class LinkCheckerTest {
8776
rateLimitGH(-1500),
8877
ok(),
8978
),
90-
"https://www.bearer.com/" to listOf(
79+
"https://www.bearer.com/" to mutableListOf(
9180
// Use variant with "delay-seconds"
9281
rateLimitSpecSec(1),
9382
// Use variant with "http-date"
9483
rateLimitSpecDate(100),
9584
ok(),
9685
),
97-
"https://github.com/androidx/androidx/blob/androidx-main/buildSrc/public/src/main/kotlin/androidx/build/LibraryGroups.kt" to listOf(
86+
"https://github.com/androidx/androidx/blob/androidx-main/buildSrc/public/src/main/kotlin/androidx/build/LibraryGroups.kt" to mutableListOf(
9887
notFound(),
9988
),
10089
),
@@ -104,7 +93,7 @@ internal class LinkCheckerTest {
10493

10594
assertThat(checker.findInvalid(client)).containsExactlyInAnyOrderEntriesOf(
10695
mapOf(
107-
"Demo.md:1:25" to "https://ya.ru -> 302 -> https://ya.ru/",
96+
"Demo.md:1:25" to "https://habr.com/ru/company/otus/blog/707724/comments -> 302 -> https://habr.com/ru/company/otus/blog/707724/comments/",
10897
"Demo.md:3:14" to "http://schema.org -> 301 -> https://schema.org/",
10998
"Demo.md:7:14" to "https://github.com/androidx/androidx/blob/androidx-main/buildSrc/public/src/main/kotlin/androidx/build/LibraryGroups.kt -> 404",
11099
),
@@ -132,72 +121,24 @@ internal class LinkCheckerTest {
132121
?: ProxySelector.getDefault()
133122
}
134123

135-
private class MockedHttpClient(
136-
private val worker: ResponseBuilder<Any?>,
137-
) : HttpClient() {
138-
override fun cookieHandler(): Optional<CookieHandler> = Optional.empty()
139-
override fun connectTimeout(): Optional<Duration> = Optional.empty()
140-
override fun followRedirects(): Redirect = Redirect.NEVER
141-
override fun proxy(): Optional<ProxySelector> = Optional.empty()
142-
override fun sslContext(): SSLContext = SSLContext.getDefault()
143-
override fun sslParameters(): SSLParameters = sslContext().defaultSSLParameters
144-
override fun authenticator(): Optional<Authenticator> = Optional.empty()
145-
override fun version(): Version = Version.HTTP_1_1
146-
override fun executor(): Optional<Executor> = Optional.empty()
147-
148-
override fun <T : Any?> sendAsync(
149-
request: HttpRequest,
150-
responseBodyHandler: HttpResponse.BodyHandler<T>,
151-
pushPromiseHandler: HttpResponse.PushPromiseHandler<T>?,
152-
): CompletableFuture<HttpResponse<T>> = sendAsync(request, responseBodyHandler)
153-
154-
override fun <T : Any?> sendAsync(
155-
request: HttpRequest,
156-
responseBodyHandler: HttpResponse.BodyHandler<T>,
157-
): CompletableFuture<HttpResponse<T>> = CompletableFuture.supplyAsync { send(request, responseBodyHandler) }
158-
159-
@Suppress("UNCHECKED_CAST")
160-
override fun <T : Any?> send(request: HttpRequest, responseBodyHandler: HttpResponse.BodyHandler<T>): HttpResponse<T> =
161-
worker(request) as HttpResponse<T>
162-
163-
companion object {
164-
fun fromMeta(responses: Map<String, List<ResponseMeta>>): HttpClient = fromBuilders(
165-
responses.mapValues {
166-
it.value
167-
.map<ResponseMeta, ResponseBuilder<Any?>> { metaBuilder ->
168-
{ req ->
169-
val meta = metaBuilder()
170-
MockedHttpResponse.fromRequest(req, meta.first, meta.second.mapValues { h -> listOf(h.value) })
171-
}
172-
}
173-
.toMutableList()
174-
},
175-
)
176-
177-
fun fromBuilders(responses: Map<String, MutableList<ResponseBuilder<Any?>>>): HttpClient = MockedHttpClient { req ->
178-
responses[req.uri().toString()]?.removeFirst()?.invoke(req) ?: fail("Unknown response builders for ${req.uri()}")
179-
}
180-
}
181-
}
182-
183-
private class MockedHttpResponse<T : Any?>(
184-
private val request: HttpRequest,
185-
private val statusCode: Int,
186-
private val headers: HttpHeaders,
187-
) : HttpResponse<T> {
188-
override fun statusCode(): Int = statusCode
189-
override fun request(): HttpRequest = request
190-
override fun previousResponse(): Optional<HttpResponse<T>> = Optional.empty()
191-
override fun headers(): HttpHeaders = headers
192-
override fun body(): T? = null
193-
override fun sslSession(): Optional<SSLSession> = Optional.empty()
194-
override fun uri(): URI = request().uri()
195-
override fun version(): HttpClient.Version = request().version().orElse(HttpClient.Version.HTTP_1_1)
196-
197-
companion object {
198-
fun <T : Any?> fromRequest(request: HttpRequest, statusCode: Int, headers: Map<String, List<String>>): HttpResponse<T> = MockedHttpResponse(
199-
request, statusCode, HttpHeaders.of(headers) { _, _ -> true },
200-
)
124+
private companion object {
125+
fun mockHttpClient(responses: Map<String, MutableList<ResponseMeta>>): OkHttpClient {
126+
val interceptor = MockInterceptor()
127+
128+
interceptor.addRule()
129+
.anyTimes()
130+
.answer { req ->
131+
val uri = req.url.toUri()
132+
val meta = ((responses[uri.toString()] ?: fail("Unknown URI: $uri")).removeFirstOrNull() ?: fail("Too many requests for URI: $uri"))()
133+
okhttp3.Response.Builder()
134+
.code(meta.first)
135+
.headers(meta.second.toHeaders())
136+
.body("")
137+
}
138+
139+
return OkHttpClient.Builder()
140+
.addInterceptor(interceptor)
141+
.build();
201142
}
202143
}
203144

‎src/test/resources/linkChecker/Demo.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
Link with name: [named](https://ya.ru).
2-
Link with name: [named](https://ya.ru/).
1+
Link with name: [named](https://habr.com/ru/company/otus/blog/707724/comments).
2+
Link with name: [named](https://habr.com/ru/company/otus/blog/707724/comments/).
33
Link inlined http://schema.org.
44
Link with rate limiting:
55
* https://github.com/androidx/androidx/blob/androidx-main/build.gradle

0 commit comments

Comments
 (0)