Add dynamic endpoints response json keys mapping

sdk-scrapper/api/sdk-scrapper.api

@@ -69,6 +69,7 @@ public final class io/github/wulkanowy/sdk/scrapper/Scrapper {
 	public static synthetic fun getReceivedMessages$default (Lio/github/wulkanowy/sdk/scrapper/Scrapper;Ljava/lang/String;IILkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object;
 	public final fun getRecipients (Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public final fun getRegisteredDevices (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
+	public final fun getResponseMapping ()Ljava/util/Map;
 	public final fun getSchool (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public final fun getSchoolId ()Ljava/lang/String;
 	public final fun getSchoolYear ()I
@@ -116,6 +117,7 @@ public final class io/github/wulkanowy/sdk/scrapper/Scrapper {
 	public final fun setLogLevel (Lokhttp3/logging/HttpLoggingInterceptor$Level;)V
 	public final fun setLoginType (Lio/github/wulkanowy/sdk/scrapper/Scrapper$LoginType;)V
 	public final fun setPassword (Ljava/lang/String;)V
+	public final fun setResponseMapping (Ljava/util/Map;)V
 	public final fun setSchoolId (Ljava/lang/String;)V
 	public final fun setSchoolYear (I)V
 	public final fun setSsl (Z)V

sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/ApiEndpoints.kt

@@ -811,6 +811,47 @@ internal val ApiEndpointsVTokenMap = mapOf(
     ),
 )
 
+internal val ApiEndpointsResponseMapping = mapOf(
+    "24.04.0010.58863" to mapOf(
+        "uonetplus-wiadomosciplus" to mapOf(
+            "__common__" to mapOf(
+                "apiGlobalKey" to "rtvrHBuCAwCEEAtIsBtuHJBtFttEtCIJ",
+                "data" to "GFBGBsJGJFHvErwAswuGruutHHtuHEuG",
+                "hasZalaczniki" to "IDJArEArvBsIErGDrsuEuGssHwsHGEts",
+                "id" to "IDwrCCJDGFrHEuIFsvEvDDJwtBFECCHJ",
+                "korespondenci" to "AurrDtEJwwrEEwJtIJvEHHwICuBBDFGF",
+                "nieprzeczytanePrzeczytanePrzez" to "DBtuuvuGvEBsEAHAJrJECsDGuFrFsuGw",
+                "przeczytana" to "FuBsJwBvHErtEDAwJGuHCuIHJwBrrJFI",
+                "skrzynka" to "AuvBvruBGtGvEJHtIrsGvrIBDGAIFJCB",
+                "temat" to "rtBCHvtsICwDEEFuJwvwJDCJBIAvAGCv",
+                "uzytkownikRola" to "twrCEIrHrsuGEIFIsCGEDJHDwrCICwBG",
+                "wazna" to "AtAsGsuEIAurEGBAJBCGIIFsuwFrwsJt",
+                "wycofana" to "twsrwAIrvGJFEutCrBrDvHCGwGHGCAGv",
+            ),
+            "Skrzynki" to mapOf(
+                "globalKey" to "uwtBBGEJtGHCEEvvsuCJEtCJHCrustHu",
+                "nazwa" to "svttEIFJuuvGECBBrGCGtuuBFJBCAtGE",
+                "typUzytkownika" to "BAICCDrJHtAEEtABrJFuFtvGuICrrCGC",
+            ),
+            "WiadomoscSzczegoly" to mapOf(
+                "apiGlobalKey" to "IvDDwGIurwIrEHDHIBvAvBEBvCsstBCC",
+                "data" to "utFsEtBJsrEuEECBJDuuIEsvFDCsEIuv",
+                "dataWycofania" to "DDAtuGHJBIEFErBArstDrsBAvIsvBHHs",
+                "id" to "vFICGIHIAwvBEwIHrvstBvwvCJJIGwJE",
+                "nadawca" to "DuJuEEvwFwAFEsAErCEtuIBtvwDsJutA",
+                "nadawcaInfo" to "DDAtuGHJBIEFErBArstDrsBAvIsvBHHs",
+                "nadawcaTyp" to "DGswFwrIGwFsEEFFrBDwvsrIsFHHCGst",
+                "odbiorcy" to "srBIECHwtJEuEJFIsBHBGsGIrHuBCJAu",
+                "odczytana" to "wwArFDurBrDFEvrAJGrsIIFADCswFuIE",
+                "temat" to "wuEwstuHDJCGEuEvsBJrEGFvIwvuvCJC",
+                "tresc" to "BEDvtCIEBCGJEDGwswIFIvIuAHFDrAJI",
+                "wycofana" to "HrvHItBsstFrEABGIDtuuGJJIGBECwFu",
+                "zalaczniki" to "GvtvvDBGvsAHEBsDsBJIJGtAtCvFswAI",
+            ),
+        ),
+    ),
+)
+
 internal val ApiEndpointsVHeaders = mapOf(
     "24.04.0003.58698" to mapOf(
         "uonetplus-wiadomosciplus" to mapOf(

sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Scrapper.kt

@@ -230,6 +230,12 @@ class Scrapper {
             vHeadersMap = value
         }
 
+    var responseMapping: Map<String, Map<String, Map<String, Map<String, String>>>>
+        get() = responseMap
+        set(value) {
+            responseMap = value
+        }
+
     var vParamsEvaluation: suspend () -> EvaluateHandler
         get() = vParamsRun
         set(value) {
@@ -240,6 +246,7 @@ class Scrapper {
         var endpointsMap: Map<String, Map<String, Map<String, String>>> = ApiEndpointsMap
         var vTokenMap: Map<String, Map<String, Map<String, String>>> = ApiEndpointsVTokenMap
         var vHeadersMap: Map<String, Map<String, Map<String, String>>> = ApiEndpointsVHeaders
+        var responseMap: Map<String, Map<String, Map<String, Map<String, String>>>> = ApiEndpointsResponseMapping
         var vParamsRun: suspend () -> EvaluateHandler = { object : EvaluateHandler {} }
     }
 

sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/Utils.kt

@@ -332,6 +332,15 @@ internal suspend fun getModuleHeadersFromDocument(document: Document): ModuleHea
     )
 }
 
+internal fun getModuleHost(url: HttpUrl): String {
+    return when {
+        MessagesModuleHost in url.host -> MessagesModuleHost
+        StudentPlusModuleHost in url.host -> StudentPlusModuleHost
+        StudentModuleHost in url.host -> StudentModuleHost
+        else -> ""
+    }
+}
+
 internal fun getVHeaders(moduleHost: String, url: HttpUrl, headers: ModuleHeaders?): Map<String, String> {
     val vHeaders = Scrapper.vHeadersMap[headers?.appVersion] ?: ApiEndpointsVHeaders[headers?.appVersion]
 

sdk-scrapper/src/main/kotlin/io/github/wulkanowy/sdk/scrapper/interceptor/AutoLoginInterceptor.kt

@@ -1,7 +1,9 @@
 package io.github.wulkanowy.sdk.scrapper.interceptor
 
+import io.github.wulkanowy.sdk.scrapper.ApiEndpointsResponseMapping
 import io.github.wulkanowy.sdk.scrapper.ApiResponse
 import io.github.wulkanowy.sdk.scrapper.CookieJarCabinet
+import io.github.wulkanowy.sdk.scrapper.Scrapper
 import io.github.wulkanowy.sdk.scrapper.Scrapper.LoginType
 import io.github.wulkanowy.sdk.scrapper.Scrapper.LoginType.ADFS
 import io.github.wulkanowy.sdk.scrapper.Scrapper.LoginType.ADFSCards
@@ -12,6 +14,8 @@ import io.github.wulkanowy.sdk.scrapper.Scrapper.LoginType.STANDARD
 import io.github.wulkanowy.sdk.scrapper.exception.VulcanClientError
 import io.github.wulkanowy.sdk.scrapper.exception.VulcanServerError
 import io.github.wulkanowy.sdk.scrapper.getModuleHeadersFromDocument
+import io.github.wulkanowy.sdk.scrapper.getModuleHost
+import io.github.wulkanowy.sdk.scrapper.getPathIndexByModuleHost
 import io.github.wulkanowy.sdk.scrapper.getVHeaders
 import io.github.wulkanowy.sdk.scrapper.isAnyMappingAvailable
 import io.github.wulkanowy.sdk.scrapper.login.LoginModuleResult
@@ -26,6 +30,11 @@ import io.github.wulkanowy.sdk.scrapper.repository.AccountRepository.Companion.S
 import io.github.wulkanowy.sdk.scrapper.repository.AccountRepository.Companion.SELECTOR_STANDARD
 import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
 import okhttp3.HttpUrl
 import okhttp3.Interceptor
 import okhttp3.MediaType
@@ -33,8 +42,10 @@ import okhttp3.Protocol
 import okhttp3.Request
 import okhttp3.Response
 import okhttp3.ResponseBody
+import okhttp3.ResponseBody.Companion.toResponseBody
 import okio.Buffer
 import okio.BufferedSource
+import okio.use
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import org.jsoup.select.Elements
@@ -72,7 +83,7 @@ internal class AutoLoginInterceptor(
             val request = chain.request()
             checkRequest()
             val response = try {
-                chain.proceed(request.attachModuleHeaders())
+                performRequest(chain, request)
             } catch (e: Throwable) {
                 when (e) {
                     is VulcanClientError -> checkHttpErrorResponse(e, url)
@@ -80,12 +91,6 @@ internal class AutoLoginInterceptor(
                 }
                 throw e
             }
-            if (response.body?.contentType()?.subtype != "json") {
-                val body = response.peekBody(Long.MAX_VALUE).byteStream()
-                val html = Jsoup.parse(body, null, url)
-                checkResponse(html, url, response)
-                saveModuleHeaders(html, uri)
-            }
             response
         } catch (e: NotLoggedInException) {
             if (loginLock.tryLock()) {
@@ -109,7 +114,7 @@ internal class AutoLoginInterceptor(
                         StudentModuleHost in uri.host -> student.getOrThrow()
                         else -> logger.info("Resource don't need further login anyway")
                     }
-                    chain.proceed(chain.request().attachModuleHeaders())
+                    performRequest(chain, chain.request())
                 } catch (e: IOException) {
                     logger.debug("IO Error occurred on login")
                     throw e
@@ -131,7 +136,7 @@ internal class AutoLoginInterceptor(
                     logger.debug("User logged in. Retry after login...")
                 }
 
-                chain.proceed(chain.request().attachModuleHeaders())
+                performRequest(chain, chain.request())
             }
         }
     }
@@ -190,6 +195,69 @@ internal class AutoLoginInterceptor(
             .build()
     }
 
+    private fun performRequest(chain: Interceptor.Chain, request: Request): Response {
+        val response = chain.proceed(request.attachModuleHeaders())
+        val url = request.url.toString()
+        val uri = request.url
+
+        return if (response.body?.contentType()?.subtype != "json") {
+            val body = response.peekBody(Long.MAX_VALUE).byteStream()
+            val html = Jsoup.parse(body, null, url)
+            checkResponse(html, url, response)
+            saveModuleHeaders(html, uri)
+            response
+        } else {
+            handleResponseMapping(response, uri)
+        }
+    }
+
+    private fun handleResponseMapping(response: Response, uri: HttpUrl): Response {
+        val moduleHost = getModuleHost(uri)
+        val pathSegmentIndex = getPathIndexByModuleHost(moduleHost)
+        val pathKey = uri.pathSegments.getOrNull(pathSegmentIndex)
+
+        val headers = headersByHost[moduleHost]
+        val mappings = Scrapper.responseMap[headers?.appVersion]?.get(moduleHost)
+            ?: ApiEndpointsResponseMapping[headers?.appVersion]?.get(moduleHost)
+        if (mappings.isNullOrEmpty()) return response
+
+        val jsonMappings = mappings[pathKey] ?: mappings["__common__"]
+
+        return response.body?.byteStream()?.bufferedReader()?.use {
+            val contentType = response.body?.contentType()
+            val body = mapResponseContent(it.readText(), jsonMappings).toResponseBody(contentType)
+            response.newBuilder().body(body).build()
+        } ?: response
+    }
+
+    private fun mapResponseContent(input: String, jsonMappings: Map<String, String>?): String {
+        return when (val response = Json.decodeFromString<JsonElement>(input)) {
+            is JsonArray -> JsonArray(
+                response.jsonArray.map {
+                    when (it) {
+                        is JsonArray -> it.jsonArray
+                        is JsonObject -> mapJsonObjectKeys(it.jsonObject, jsonMappings)
+                        else -> it
+                    }
+                },
+            )
+
+            is JsonObject -> mapJsonObjectKeys(response.jsonObject, jsonMappings)
+            else -> response
+        }.toString()
+    }
+
+    private fun mapJsonObjectKeys(jsonObject: JsonObject, jsonMappings: Map<String, String>?): JsonObject {
+        val mapping = jsonMappings?.map { (key, value) ->
+            value to key
+        }.orEmpty().toMap()
+        return JsonObject(
+            jsonObject.mapKeys {
+                mapping[it.key] ?: it.key
+            },
+        )
+    }
+
     private fun checkRequest() {
         if (emptyCookieJarIntercept && !cookieJarCabinet.isUserCookiesExist()) {
             throw NotLoggedInException("No cookie found! You are not logged in yet")

sdk/api/sdk.api

@@ -62,6 +62,7 @@ public final class io/github/wulkanowy/sdk/Sdk {
 	public static synthetic fun getReceivedMessages$default (Lio/github/wulkanowy/sdk/Sdk;Ljava/lang/String;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object;
 	public final fun getRecipients (Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public final fun getRegisteredDevices (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
+	public final fun getResponseMapping ()Ljava/util/Map;
 	public final fun getSchool (Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public final fun getSchoolSymbol ()Ljava/lang/String;
 	public final fun getSchoolYear ()I
@@ -116,6 +117,7 @@ public final class io/github/wulkanowy/sdk/Sdk {
 	public final fun setMode (Lio/github/wulkanowy/sdk/Sdk$Mode;)V
 	public final fun setPassword (Ljava/lang/String;)V
 	public final fun setPrivatePem (Ljava/lang/String;)V
+	public final fun setResponseMapping (Ljava/util/Map;)V
 	public final fun setSchoolSymbol (Ljava/lang/String;)V
 	public final fun setSchoolYear (I)V
 	public final fun setScrapperBaseUrl (Ljava/lang/String;)V

sdk/src/main/kotlin/io/github/wulkanowy/sdk/Sdk.kt

@@ -254,6 +254,12 @@ class Sdk {
             scrapper.vHeaders = value
         }
 
+    var responseMapping
+        get() = scrapper.responseMapping
+        set(value) {
+            scrapper.responseMapping = value
+        }
+
     var vParamsEvaluation: suspend () -> EvaluateHandler
         get() = scrapper.vParamsEvaluation
         set(value) {