From 7f9456fe6c0a6915f1a5350c95637aca430ece56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Pich?= Date: Thu, 30 May 2024 15:32:15 +0200 Subject: [PATCH] Add dynamic endpoints response json keys mapping --- sdk-scrapper/api/sdk-scrapper.api | 2 + .../wulkanowy/sdk/scrapper/ApiEndpoints.kt | 41 +++++++++ .../github/wulkanowy/sdk/scrapper/Scrapper.kt | 7 ++ .../io/github/wulkanowy/sdk/scrapper/Utils.kt | 9 ++ .../interceptor/AutoLoginInterceptor.kt | 86 +++++++++++++++++-- sdk/api/sdk.api | 2 + .../kotlin/io/github/wulkanowy/sdk/Sdk.kt | 6 ++ 7 files changed, 144 insertions(+), 9 deletions(-) diff --git a/sdk-scrapper/api/sdk-scrapper.api b/sdk-scrapper/api/sdk-scrapper.api index 77e7df03..cd7502e2 100644 --- a/sdk-scrapper/api/sdk-scrapper.api +++ b/sdk-scrapper/api/sdk-scrapper.api @@ -69,6 +69,7 @@ public final class io/github/wulkanowy/sdk/scrapper/Scrapper { public static synthetic fun getReceivedMessages$default (Lio/github/wulkanowy/sdk/scrapper/Scrapper;Ljava/lang/String;IILkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; public final fun getRecipients (Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun getRegisteredDevices (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public final fun getResponseMapping ()Ljava/util/Map; public final fun getSchool (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun getSchoolId ()Ljava/lang/String; public final fun getSchoolYear ()I @@ -116,6 +117,7 @@ public final class io/github/wulkanowy/sdk/scrapper/Scrapper { public final fun setLogLevel (Lokhttp3/logging/HttpLoggingInterceptor$Level;)V public final fun setLoginType (Lio/github/wulkanowy/sdk/scrapper/Scrapper$LoginType;)V public final fun setPassword (Ljava/lang/String;)V + public final fun setResponseMapping (Ljava/util/Map;)V public final fun setSchoolId (Ljava/lang/String;)V public final fun setSchoolYear (I)V public final fun setSsl (Z)V diff --git a/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/ApiEndpoints.kt b/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/ApiEndpoints.kt index 5ece57f8..ef56e4af 100644 --- a/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/ApiEndpoints.kt +++ b/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/ApiEndpoints.kt @@ -811,6 +811,47 @@ internal val ApiEndpointsVTokenMap = mapOf( ), ) +internal val ApiEndpointsResponseMapping = mapOf( + "24.04.0010.58863" to mapOf( + "uonetplus-wiadomosciplus" to mapOf( + "__common__" to mapOf( + "apiGlobalKey" to "rtvrHBuCAwCEEAtIsBtuHJBtFttEtCIJ", + "data" to "GFBGBsJGJFHvErwAswuGruutHHtuHEuG", + "hasZalaczniki" to "IDJArEArvBsIErGDrsuEuGssHwsHGEts", + "id" to "IDwrCCJDGFrHEuIFsvEvDDJwtBFECCHJ", + "korespondenci" to "AurrDtEJwwrEEwJtIJvEHHwICuBBDFGF", + "nieprzeczytanePrzeczytanePrzez" to "DBtuuvuGvEBsEAHAJrJECsDGuFrFsuGw", + "przeczytana" to "FuBsJwBvHErtEDAwJGuHCuIHJwBrrJFI", + "skrzynka" to "AuvBvruBGtGvEJHtIrsGvrIBDGAIFJCB", + "temat" to "rtBCHvtsICwDEEFuJwvwJDCJBIAvAGCv", + "uzytkownikRola" to "twrCEIrHrsuGEIFIsCGEDJHDwrCICwBG", + "wazna" to "AtAsGsuEIAurEGBAJBCGIIFsuwFrwsJt", + "wycofana" to "twsrwAIrvGJFEutCrBrDvHCGwGHGCAGv", + ), + "Skrzynki" to mapOf( + "globalKey" to "uwtBBGEJtGHCEEvvsuCJEtCJHCrustHu", + "nazwa" to "svttEIFJuuvGECBBrGCGtuuBFJBCAtGE", + "typUzytkownika" to "BAICCDrJHtAEEtABrJFuFtvGuICrrCGC", + ), + "WiadomoscSzczegoly" to mapOf( + "apiGlobalKey" to "IvDDwGIurwIrEHDHIBvAvBEBvCsstBCC", + "data" to "utFsEtBJsrEuEECBJDuuIEsvFDCsEIuv", + "dataWycofania" to "DDAtuGHJBIEFErBArstDrsBAvIsvBHHs", + "id" to "vFICGIHIAwvBEwIHrvstBvwvCJJIGwJE", + "nadawca" to "DuJuEEvwFwAFEsAErCEtuIBtvwDsJutA", + "nadawcaInfo" to "DDAtuGHJBIEFErBArstDrsBAvIsvBHHs", + "nadawcaTyp" to "DGswFwrIGwFsEEFFrBDwvsrIsFHHCGst", + "odbiorcy" to "srBIECHwtJEuEJFIsBHBGsGIrHuBCJAu", + "odczytana" to "wwArFDurBrDFEvrAJGrsIIFADCswFuIE", + "temat" to "wuEwstuHDJCGEuEvsBJrEGFvIwvuvCJC", + "tresc" to "BEDvtCIEBCGJEDGwswIFIvIuAHFDrAJI", + "wycofana" to "HrvHItBsstFrEABGIDtuuGJJIGBECwFu", + "zalaczniki" to "GvtvvDBGvsAHEBsDsBJIJGtAtCvFswAI", + ), + ), + ), +) + internal val ApiEndpointsVHeaders = mapOf( "24.04.0003.58698" to mapOf( "uonetplus-wiadomosciplus" to mapOf( diff --git a/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Scrapper.kt b/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Scrapper.kt index 6ea9a4df..34d719cb 100644 --- a/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Scrapper.kt +++ b/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Scrapper.kt @@ -230,6 +230,12 @@ class Scrapper { vHeadersMap = value } + var responseMapping: Map>>> + get() = responseMap + set(value) { + responseMap = value + } + var vParamsEvaluation: suspend () -> EvaluateHandler get() = vParamsRun set(value) { @@ -240,6 +246,7 @@ class Scrapper { var endpointsMap: Map>> = ApiEndpointsMap var vTokenMap: Map>> = ApiEndpointsVTokenMap var vHeadersMap: Map>> = ApiEndpointsVHeaders + var responseMap: Map>>> = ApiEndpointsResponseMapping var vParamsRun: suspend () -> EvaluateHandler = { object : EvaluateHandler {} } } diff --git a/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Utils.kt b/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Utils.kt index cedbca74..a2ae2327 100644 --- a/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Utils.kt +++ b/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Utils.kt @@ -332,6 +332,15 @@ internal suspend fun getModuleHeadersFromDocument(document: Document): ModuleHea ) } +internal fun getModuleHost(url: HttpUrl): String { + return when { + MessagesModuleHost in url.host -> MessagesModuleHost + StudentPlusModuleHost in url.host -> StudentPlusModuleHost + StudentModuleHost in url.host -> StudentModuleHost + else -> "" + } +} + internal fun getVHeaders(moduleHost: String, url: HttpUrl, headers: ModuleHeaders?): Map { val vHeaders = Scrapper.vHeadersMap[headers?.appVersion] ?: ApiEndpointsVHeaders[headers?.appVersion] diff --git a/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/interceptor/AutoLoginInterceptor.kt b/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/interceptor/AutoLoginInterceptor.kt index 040ce479..9d7c35af 100644 --- a/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/interceptor/AutoLoginInterceptor.kt +++ b/sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/interceptor/AutoLoginInterceptor.kt @@ -1,7 +1,9 @@ package io.github.wulkanowy.sdk.scrapper.interceptor +import io.github.wulkanowy.sdk.scrapper.ApiEndpointsResponseMapping import io.github.wulkanowy.sdk.scrapper.ApiResponse import io.github.wulkanowy.sdk.scrapper.CookieJarCabinet +import io.github.wulkanowy.sdk.scrapper.Scrapper import io.github.wulkanowy.sdk.scrapper.Scrapper.LoginType import io.github.wulkanowy.sdk.scrapper.Scrapper.LoginType.ADFS import io.github.wulkanowy.sdk.scrapper.Scrapper.LoginType.ADFSCards @@ -12,6 +14,8 @@ import io.github.wulkanowy.sdk.scrapper.Scrapper.LoginType.STANDARD import io.github.wulkanowy.sdk.scrapper.exception.VulcanClientError import io.github.wulkanowy.sdk.scrapper.exception.VulcanServerError import io.github.wulkanowy.sdk.scrapper.getModuleHeadersFromDocument +import io.github.wulkanowy.sdk.scrapper.getModuleHost +import io.github.wulkanowy.sdk.scrapper.getPathIndexByModuleHost import io.github.wulkanowy.sdk.scrapper.getVHeaders import io.github.wulkanowy.sdk.scrapper.isAnyMappingAvailable import io.github.wulkanowy.sdk.scrapper.login.LoginModuleResult @@ -26,6 +30,11 @@ import io.github.wulkanowy.sdk.scrapper.repository.AccountRepository.Companion.S import io.github.wulkanowy.sdk.scrapper.repository.AccountRepository.Companion.SELECTOR_STANDARD import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject import okhttp3.HttpUrl import okhttp3.Interceptor import okhttp3.MediaType @@ -33,8 +42,10 @@ import okhttp3.Protocol import okhttp3.Request import okhttp3.Response import okhttp3.ResponseBody +import okhttp3.ResponseBody.Companion.toResponseBody import okio.Buffer import okio.BufferedSource +import okio.use import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.select.Elements @@ -72,7 +83,7 @@ internal class AutoLoginInterceptor( val request = chain.request() checkRequest() val response = try { - chain.proceed(request.attachModuleHeaders()) + performRequest(chain, request) } catch (e: Throwable) { when (e) { is VulcanClientError -> checkHttpErrorResponse(e, url) @@ -80,12 +91,6 @@ internal class AutoLoginInterceptor( } throw e } - if (response.body?.contentType()?.subtype != "json") { - val body = response.peekBody(Long.MAX_VALUE).byteStream() - val html = Jsoup.parse(body, null, url) - checkResponse(html, url, response) - saveModuleHeaders(html, uri) - } response } catch (e: NotLoggedInException) { if (loginLock.tryLock()) { @@ -109,7 +114,7 @@ internal class AutoLoginInterceptor( StudentModuleHost in uri.host -> student.getOrThrow() else -> logger.info("Resource don't need further login anyway") } - chain.proceed(chain.request().attachModuleHeaders()) + performRequest(chain, chain.request()) } catch (e: IOException) { logger.debug("IO Error occurred on login") throw e @@ -131,7 +136,7 @@ internal class AutoLoginInterceptor( logger.debug("User logged in. Retry after login...") } - chain.proceed(chain.request().attachModuleHeaders()) + performRequest(chain, chain.request()) } } } @@ -190,6 +195,69 @@ internal class AutoLoginInterceptor( .build() } + private fun performRequest(chain: Interceptor.Chain, request: Request): Response { + val response = chain.proceed(request.attachModuleHeaders()) + val url = request.url.toString() + val uri = request.url + + return if (response.body?.contentType()?.subtype != "json") { + val body = response.peekBody(Long.MAX_VALUE).byteStream() + val html = Jsoup.parse(body, null, url) + checkResponse(html, url, response) + saveModuleHeaders(html, uri) + response + } else { + handleResponseMapping(response, uri) + } + } + + private fun handleResponseMapping(response: Response, uri: HttpUrl): Response { + val moduleHost = getModuleHost(uri) + val pathSegmentIndex = getPathIndexByModuleHost(moduleHost) + val pathKey = uri.pathSegments.getOrNull(pathSegmentIndex) + + val headers = headersByHost[moduleHost] + val mappings = Scrapper.responseMap[headers?.appVersion]?.get(moduleHost) + ?: ApiEndpointsResponseMapping[headers?.appVersion]?.get(moduleHost) + if (mappings.isNullOrEmpty()) return response + + val jsonMappings = mappings[pathKey] ?: mappings["__common__"] + + return response.body?.byteStream()?.bufferedReader()?.use { + val contentType = response.body?.contentType() + val body = mapResponseContent(it.readText(), jsonMappings).toResponseBody(contentType) + response.newBuilder().body(body).build() + } ?: response + } + + private fun mapResponseContent(input: String, jsonMappings: Map?): String { + return when (val response = Json.decodeFromString(input)) { + is JsonArray -> JsonArray( + response.jsonArray.map { + when (it) { + is JsonArray -> it.jsonArray + is JsonObject -> mapJsonObjectKeys(it.jsonObject, jsonMappings) + else -> it + } + }, + ) + + is JsonObject -> mapJsonObjectKeys(response.jsonObject, jsonMappings) + else -> response + }.toString() + } + + private fun mapJsonObjectKeys(jsonObject: JsonObject, jsonMappings: Map?): JsonObject { + val mapping = jsonMappings?.map { (key, value) -> + value to key + }.orEmpty().toMap() + return JsonObject( + jsonObject.mapKeys { + mapping[it.key] ?: it.key + }, + ) + } + private fun checkRequest() { if (emptyCookieJarIntercept && !cookieJarCabinet.isUserCookiesExist()) { throw NotLoggedInException("No cookie found! You are not logged in yet") diff --git a/sdk/api/sdk.api b/sdk/api/sdk.api index 294fd9fd..bf80c414 100644 --- a/sdk/api/sdk.api +++ b/sdk/api/sdk.api @@ -62,6 +62,7 @@ public final class io/github/wulkanowy/sdk/Sdk { public static synthetic fun getReceivedMessages$default (Lio/github/wulkanowy/sdk/Sdk;Ljava/lang/String;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; public final fun getRecipients (Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun getRegisteredDevices (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public final fun getResponseMapping ()Ljava/util/Map; public final fun getSchool (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun getSchoolSymbol ()Ljava/lang/String; public final fun getSchoolYear ()I @@ -116,6 +117,7 @@ public final class io/github/wulkanowy/sdk/Sdk { public final fun setMode (Lio/github/wulkanowy/sdk/Sdk$Mode;)V public final fun setPassword (Ljava/lang/String;)V public final fun setPrivatePem (Ljava/lang/String;)V + public final fun setResponseMapping (Ljava/util/Map;)V public final fun setSchoolSymbol (Ljava/lang/String;)V public final fun setSchoolYear (I)V public final fun setScrapperBaseUrl (Ljava/lang/String;)V diff --git a/sdk/src/main/kotlin/io/github/wulkanowy/sdk/Sdk.kt b/sdk/src/main/kotlin/io/github/wulkanowy/sdk/Sdk.kt index 8fe3eadc..aa4887e0 100644 --- a/sdk/src/main/kotlin/io/github/wulkanowy/sdk/Sdk.kt +++ b/sdk/src/main/kotlin/io/github/wulkanowy/sdk/Sdk.kt @@ -254,6 +254,12 @@ class Sdk { scrapper.vHeaders = value } + var responseMapping + get() = scrapper.responseMapping + set(value) { + scrapper.responseMapping = value + } + var vParamsEvaluation: suspend () -> EvaluateHandler get() = scrapper.vParamsEvaluation set(value) {