diff --git a/build.gradle.kts b/build.gradle.kts index 6e3ad7c..e71f15a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -90,6 +90,8 @@ dependencies { implementation("com.sendgrid:sendgrid-java:4.4.1") implementation("io.lettuce:lettuce-core:5.2.2.RELEASE") implementation("com.rabbitmq:amqp-client:5.8.0") + implementation("org.elasticsearch.client:elasticsearch-rest-client:6.7.1") + implementation("com.jayway.jsonpath:json-path:2.4.0") testImplementation("io.ktor:ktor-server-tests:$ktor_version") testImplementation("io.ktor:ktor-client-mock:$ktor_version") diff --git a/src/main/kotlin/fr/dcproject/Application.kt b/src/main/kotlin/fr/dcproject/Application.kt index c13f748..d15fcdb 100644 --- a/src/main/kotlin/fr/dcproject/Application.kt +++ b/src/main/kotlin/fr/dcproject/Application.kt @@ -36,6 +36,8 @@ import io.ktor.routing.Routing import io.ktor.util.KtorExperimentalAPI import io.ktor.websocket.WebSockets import org.eclipse.jetty.util.log.Slf4jLog +import org.elasticsearch.client.Request +import org.elasticsearch.client.RestClient import org.koin.core.qualifier.named import org.koin.ktor.ext.Koin import org.koin.ktor.ext.get @@ -169,6 +171,57 @@ fun Application.module(env: Env = PROD) { } } + /* Create index if not exist */ + get().run { + if (performRequest(Request("HEAD", "/views?include_type_name=false")).statusLine.statusCode == 404) { + Request( + "PUT", + "/views?include_type_name=false" + ).apply { + //language=JSON + setJsonEntity( + """ + { + "settings": { + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "logged": { + "type": "boolean" + }, + "type": { + "type": "keyword" + }, + "user_ref": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "version_id": { + "type": "keyword" + }, + "ip": { + "type": "keyword" + }, + "citizen_id": { + "type": "keyword" + }, + "view_at": { + "type": "date" + } + } + } + } + """.trimIndent() + ) + }.let { + performRequest(it) + } + } + } + install(WebSockets) { pingPeriod = Duration.ofSeconds(60) // Disabled (null) by default timeout = Duration.ofSeconds(15) @@ -233,7 +286,7 @@ fun Application.module(env: Env = PROD) { install(Routing) { // trace { application.log.trace(it.buildText()) } authenticate(optional = true) { - article(get()) + article(get(), get()) auth(get(), get(), get()) citizen(get(), get()) constitution(get()) diff --git a/src/main/kotlin/fr/dcproject/ApplicationContext.kt b/src/main/kotlin/fr/dcproject/ApplicationContext.kt index 349ce2d..f90cfe3 100644 --- a/src/main/kotlin/fr/dcproject/ApplicationContext.kt +++ b/src/main/kotlin/fr/dcproject/ApplicationContext.kt @@ -6,7 +6,6 @@ import io.ktor.application.ApplicationCall import io.ktor.auth.authentication import io.ktor.util.AttributeKey import io.ktor.util.pipeline.PipelineContext -import kotlinx.coroutines.runBlocking import org.koin.core.context.GlobalContext import fr.dcproject.entity.Citizen as CitizenEntity import fr.dcproject.repository.Citizen as CitizenRepository @@ -15,11 +14,15 @@ private val citizenAttributeKey = AttributeKey("CitizenContext") val ApplicationCall.citizen: CitizenEntity get() = attributes.computeIfAbsent(citizenAttributeKey) { - runBlocking { - val user = authentication.principal() ?: throw ForbiddenException() - GlobalContext.get().koin.get().findByUser(user) - ?: throw ForbiddenException("Citizen not found for this user id \"${user.id}\"") - } + val user = authentication.principal() ?: throw ForbiddenException() + GlobalContext.get().koin.get().findByUser(user) + ?: throw ForbiddenException("Citizen not found for this user id \"${user.id}\"") + } + +val ApplicationCall.citizenOrNull: CitizenEntity? + get() = authentication.principal()?.let { + GlobalContext.get().koin.get().findByUser(it) } val PipelineContext.citizen get() = context.citizen +val PipelineContext.citizenOrNull get() = context.citizenOrNull diff --git a/src/main/kotlin/fr/dcproject/Module.kt b/src/main/kotlin/fr/dcproject/Module.kt index 6f30cf5..995b659 100644 --- a/src/main/kotlin/fr/dcproject/Module.kt +++ b/src/main/kotlin/fr/dcproject/Module.kt @@ -10,6 +10,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.rabbitmq.client.ConnectionFactory import fr.dcproject.messages.Mailer import fr.dcproject.messages.SsoManager +import fr.dcproject.views.ArticleViewManager import fr.postgresjson.connexion.Connection import fr.postgresjson.connexion.Requester import fr.postgresjson.migration.Migrations @@ -18,6 +19,8 @@ import io.ktor.client.features.websocket.WebSockets import io.ktor.util.KtorExperimentalAPI import io.lettuce.core.RedisClient import io.lettuce.core.api.async.RedisAsyncCommands +import org.apache.http.HttpHost +import org.elasticsearch.client.RestClient import org.koin.core.qualifier.named import org.koin.dsl.module import fr.dcproject.repository.Article as ArticleRepository @@ -109,6 +112,15 @@ val Module = module { single { OpinionChoiceRepository(get()) } single { OpinionArticleRepository(get()) } + // Elasticsearch Client + single { + RestClient.builder( + HttpHost("localhost", 9200, "http") + ).build() + } + + single { ArticleViewManager(get()) } + // Mailler single { Mailer(config.sendGridKey) } diff --git a/src/main/kotlin/fr/dcproject/entity/Article.kt b/src/main/kotlin/fr/dcproject/entity/Article.kt index 7d34277..dda85fe 100644 --- a/src/main/kotlin/fr/dcproject/entity/Article.kt +++ b/src/main/kotlin/fr/dcproject/entity/Article.kt @@ -18,7 +18,8 @@ class Article( override var lastVersion: Boolean = false, createdBy: CitizenBasic ) : ArticleFull, - ArticleBasic(id, title, anonymous, content, description, tags, createdBy) + ArticleBasic(id, title, anonymous, content, description, tags, createdBy), + Viewable by ViewableImp() open class ArticleBasic( id: UUID = UUID.randomUUID(), @@ -41,14 +42,20 @@ open class ArticleSimple( override var title: String, override val createdBy: CitizenBasic ) : ArticleSimpleI, - ArticleRef(id), + ArticleRefVersioning(id), EntityCreatedAt by EntityCreatedAtImp(), EntityCreatedBy by EntityCreatedByImp(createdBy), EntityDeletedAt by EntityDeletedAtImp(), - EntityVersioning by UuidEntityVersioning(), Votable by VotableImp(), Opinionable by OpinionableImp() +open class ArticleRefVersioning( + id: UUID = UUID.randomUUID(), + versionNumber: Int? = null, + versionId: UUID = UUID.randomUUID() +) : ArticleRef(id), + EntityVersioning by UuidEntityVersioning(versionNumber, versionId) + open class ArticleRef( id: UUID = UUID.randomUUID() ) : ArticleI, TargetRef(id) diff --git a/src/main/kotlin/fr/dcproject/entity/ViewAggregation.kt b/src/main/kotlin/fr/dcproject/entity/ViewAggregation.kt new file mode 100644 index 0000000..9b810e7 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/entity/ViewAggregation.kt @@ -0,0 +1,13 @@ +package fr.dcproject.entity + +import fr.postgresjson.entity.EntityI +import fr.postgresjson.entity.immutable.EntityUpdatedAt +import fr.postgresjson.entity.immutable.EntityUpdatedAtImp + +open class ViewAggregation( + val total: Int, + val unique: Int +) : EntityI, + EntityUpdatedAt by EntityUpdatedAtImp() { + constructor() : this(0, 0) +} diff --git a/src/main/kotlin/fr/dcproject/entity/Viewable.kt b/src/main/kotlin/fr/dcproject/entity/Viewable.kt new file mode 100644 index 0000000..ddbb8b0 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/entity/Viewable.kt @@ -0,0 +1,9 @@ +package fr.dcproject.entity + +interface Viewable { + var views: ViewAggregation +} + +class ViewableImp : Viewable { + override var views: ViewAggregation = ViewAggregation() +} \ No newline at end of file diff --git a/src/main/kotlin/fr/dcproject/event/EventSubscriber.kt b/src/main/kotlin/fr/dcproject/event/EventSubscriber.kt index 0661b42..0abd3f8 100644 --- a/src/main/kotlin/fr/dcproject/event/EventSubscriber.kt +++ b/src/main/kotlin/fr/dcproject/event/EventSubscriber.kt @@ -20,7 +20,7 @@ abstract class Event( } } -abstract class EntityEvent( +open class EntityEvent( val target: UuidEntity, type: String, val action: String diff --git a/src/main/kotlin/fr/dcproject/routes/Article.kt b/src/main/kotlin/fr/dcproject/routes/Article.kt index b76ffa9..21001ce 100644 --- a/src/main/kotlin/fr/dcproject/routes/Article.kt +++ b/src/main/kotlin/fr/dcproject/routes/Article.kt @@ -1,11 +1,13 @@ package fr.dcproject.routes import fr.dcproject.citizen +import fr.dcproject.citizenOrNull import fr.dcproject.event.ArticleUpdate import fr.dcproject.repository.Article.Filter import fr.dcproject.security.voter.ArticleVoter.Action.CREATE import fr.dcproject.security.voter.ArticleVoter.Action.VIEW import fr.dcproject.security.voter.assertCan +import fr.dcproject.views.ArticleViewManager import fr.postgresjson.repository.RepositoryI import io.ktor.application.application import io.ktor.application.call @@ -16,6 +18,7 @@ import io.ktor.locations.post import io.ktor.request.receive import io.ktor.response.respond import io.ktor.routing.Route +import kotlinx.coroutines.launch import fr.dcproject.entity.Article as ArticleEntity import fr.dcproject.entity.request.Article as ArticleEntityRequest import fr.dcproject.repository.Article as ArticleRepository @@ -56,7 +59,7 @@ object ArticlesPaths { } @KtorExperimentalLocationsAPI -fun Route.article(repo: ArticleRepository) { +fun Route.article(repo: ArticleRepository, viewManager: ArticleViewManager) { get { val articles = repo.find(it.page, it.limit, it.sort, it.direction, it.search, Filter(createdById = it.createdBy)) @@ -67,7 +70,13 @@ fun Route.article(repo: ArticleRepository) { get { assertCan(VIEW, it.article) + it.article.views = viewManager.getViewsCount(it.article) + call.respond(it.article) + + launch { + viewManager.addView(call.request.local.remoteHost, it.article, citizenOrNull) + } } get { diff --git a/src/main/kotlin/fr/dcproject/utils/DateTime.kt b/src/main/kotlin/fr/dcproject/utils/DateTime.kt new file mode 100644 index 0000000..b720614 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/utils/DateTime.kt @@ -0,0 +1,6 @@ +package fr.dcproject.utils + +import org.joda.time.DateTime +import org.joda.time.format.ISODateTimeFormat + +fun DateTime.toIso() = ISODateTimeFormat.dateTime().print(this) \ No newline at end of file diff --git a/src/main/kotlin/fr/dcproject/utils/Elastic.kt b/src/main/kotlin/fr/dcproject/utils/Elastic.kt new file mode 100644 index 0000000..3962a2c --- /dev/null +++ b/src/main/kotlin/fr/dcproject/utils/Elastic.kt @@ -0,0 +1,29 @@ +package fr.dcproject.utils + +import com.jayway.jsonpath.JsonPath +import com.jayway.jsonpath.PathNotFoundException +import org.apache.http.util.EntityUtils +import org.elasticsearch.client.Response +import org.slf4j.LoggerFactory + +fun Response.contentToString(): String { + return EntityUtils.toString(this.entity) +} + +fun Response.getField(jsonPath: String): Int? { + return try { + JsonPath.read(this.contentToString(), jsonPath) + } catch (e: PathNotFoundException) { + null + } +} + +fun String.getJsonField(jsonPath: String): Int? { + return try { + JsonPath.read(this, jsonPath) + } catch (e: PathNotFoundException) { + LoggerFactory.getLogger("fr.dcproject.utils.getJsonField") + .warn("No value for Json path ${JsonPath.compile(jsonPath).path}") + null + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/dcproject/views/ArticleViewManager.kt b/src/main/kotlin/fr/dcproject/views/ArticleViewManager.kt new file mode 100644 index 0000000..ef5c9e7 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/views/ArticleViewManager.kt @@ -0,0 +1,86 @@ +package fr.dcproject.views + +import fr.dcproject.entity.* +import fr.dcproject.utils.contentToString +import fr.dcproject.utils.getJsonField +import fr.dcproject.utils.toIso +import io.ktor.util.sha1 +import org.elasticsearch.client.Request +import org.elasticsearch.client.Response +import org.elasticsearch.client.RestClient +import org.joda.time.DateTime +import java.util.* + +class ArticleViewManager(private val restClient: RestClient) : ViewManager { + override fun addView(ip: String, article: ArticleRefVersioning, citizen: CitizenRef?, dateTime: DateTime): Response? { + val isLogged = (citizen != null).toString() + sha1("plop".toByteArray()) + val ref = citizen?.id ?: UUID.nameUUIDFromBytes(ip.toByteArray())!! + val request = Request( + "POST", + "/views/_doc/" + ).apply { + //language=JSON + setJsonEntity(""" + { + "logged": $isLogged, + "type": "article", + "user_ref": "$ref", + "ip": "$ip", + "id": "${article.id}", + "version_id": "${article.versionId}", + "citizen_id": "${citizen?.id}", + "view_at": "${dateTime.toIso()}" + } + """.trimIndent()) + } + + return restClient.performRequest(request) + } + + override fun getViewsCount(article: ArticleRefVersioning): ViewAggregation { + val request = Request( + "GET", + "/views/_search" + ).apply { + //language=JSON + setJsonEntity(""" + { + "size": 0, + "query": { + "bool": { + "must": { + "term": { + "version_id": "${article.versionId}" + } + } + } + }, + "aggs" : { + "total": { + "composite" : { + "sources" : [ + { "version_id": { "terms": {"field": "version_id" } } } + ] + } + }, + "unique" : { + "cardinality" : { + "field" : "user_ref", + "precision_threshold": 1 + } + } + } + } + """.trimIndent()) + } + + return restClient + .performRequest(request).contentToString().run { + ViewAggregation( + getJsonField("$.aggregations.total.buckets[0].doc_count") ?: 0, + getJsonField("$.aggregations.unique.value") ?: 0 + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/dcproject/views/ViewManager.kt b/src/main/kotlin/fr/dcproject/views/ViewManager.kt new file mode 100644 index 0000000..95a0887 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/views/ViewManager.kt @@ -0,0 +1,11 @@ +package fr.dcproject.views + +import fr.dcproject.entity.CitizenRef +import fr.dcproject.entity.ViewAggregation +import org.elasticsearch.client.Response +import org.joda.time.DateTime + +interface ViewManager { + fun addView(ip: String, entity: T, citizen: CitizenRef? = null, dateTime: DateTime = DateTime.now()): Response? + fun getViewsCount(entity: T): ViewAggregation +} \ No newline at end of file diff --git a/src/main/resources/openapi.yaml b/src/main/resources/openapi.yaml index cbc81a5..c9e3cb4 100644 --- a/src/main/resources/openapi.yaml +++ b/src/main/resources/openapi.yaml @@ -258,6 +258,9 @@ paths: parameters: - $ref: '#/components/parameters/article' get: + security: + - {} + - JWTAuth: [] summary: Get one article tags: - article diff --git a/src/test/kotlin/RunCucumberTest.kt b/src/test/kotlin/RunCucumberTest.kt index 79f3e0a..ccb2f1e 100644 --- a/src/test/kotlin/RunCucumberTest.kt +++ b/src/test/kotlin/RunCucumberTest.kt @@ -25,7 +25,7 @@ var unitialized: Boolean = false @KtorExperimentalAPI @KtorExperimentalLocationsAPI @RunWith(Cucumber::class) -@CucumberOptions(plugin = ["pretty"]) +@CucumberOptions(plugin = ["pretty"], strict = true) class RunCucumberTest : En, KoinTest { private val logger: Logger? by LoggerDelegate() diff --git a/src/test/kotlin/ViewTest.kt b/src/test/kotlin/ViewTest.kt new file mode 100644 index 0000000..6e903e6 --- /dev/null +++ b/src/test/kotlin/ViewTest.kt @@ -0,0 +1,68 @@ +import fr.dcproject.Env +import fr.dcproject.entity.ArticleRefVersioning +import fr.dcproject.entity.CitizenRef +import fr.dcproject.module +import fr.dcproject.views.ArticleViewManager +import io.ktor.locations.KtorExperimentalLocationsAPI +import io.ktor.server.testing.withTestApplication +import io.ktor.util.KtorExperimentalAPI +import org.amshove.kluent.`should equal` +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS +import org.koin.ktor.ext.get +import java.util.* + +@KtorExperimentalLocationsAPI +@KtorExperimentalAPI +@TestInstance(PER_CLASS) +class ViewTest { + @Test + fun `test View Article`() { + val article = ArticleRefVersioning(id = UUID.randomUUID(), versionId = UUID.randomUUID()) + val citizenRef = CitizenRef() + + withTestApplication({ module(Env.TEST) }) { + val viewManager: ArticleViewManager = application.get() + + /* Get view before */ + val startView = viewManager.getViewsCount(article) + + /* Add View */ + viewManager.addView( + "1.2.3.4", + article, + citizenRef + ) + + /* Add View */ + viewManager.addView( + "10.10.10.10", + article, + citizenRef + ) + + /* Add View */ + viewManager.addView( + "8.8.8.8", + article + ) + + /* Add View */ + viewManager.addView( + "1.1.1.1", + article + ) + + /* Sleep because ES is not sync ! */ + Thread.sleep(1000) + + /* Get view */ + val afterView = viewManager.getViewsCount(article) + + /* Check if view has increment */ + afterView.total `should equal` startView.total + 4 + afterView.unique `should equal` startView.unique + 3 + } + } +} diff --git a/src/test/resources/feature/articles.feature b/src/test/resources/feature/articles.feature index 5bdd35d..f18fcd0 100644 --- a/src/test/resources/feature/articles.feature +++ b/src/test/resources/feature/articles.feature @@ -1,3 +1,4 @@ +@article Feature: articles routes Scenario: The route for get articles must response a 200