From 90e7e0254d50fd41b5a5edfef44b87ca9c9ce256 Mon Sep 17 00:00:00 2001 From: Fabrice Lecomte Date: Wed, 11 Mar 2020 12:55:39 +0100 Subject: [PATCH] #15 Count total view Create Viewable interface Set Article as Viewable Create ES client Now view article increment view count Create index on start application add an extention in ApplicationCall, to get citizen or return null --- build.gradle.kts | 2 + src/main/kotlin/fr/dcproject/Application.kt | 55 +++++++++++- .../kotlin/fr/dcproject/ApplicationContext.kt | 15 ++-- src/main/kotlin/fr/dcproject/Module.kt | 12 +++ .../kotlin/fr/dcproject/entity/Article.kt | 13 ++- .../fr/dcproject/entity/ViewAggregation.kt | 13 +++ .../kotlin/fr/dcproject/entity/Viewable.kt | 9 ++ .../fr/dcproject/event/EventSubscriber.kt | 2 +- .../kotlin/fr/dcproject/routes/Article.kt | 11 ++- .../kotlin/fr/dcproject/utils/DateTime.kt | 6 ++ src/main/kotlin/fr/dcproject/utils/Elastic.kt | 29 +++++++ .../fr/dcproject/views/ArticleViewManager.kt | 86 +++++++++++++++++++ .../kotlin/fr/dcproject/views/ViewManager.kt | 11 +++ src/main/resources/openapi.yaml | 3 + src/test/kotlin/RunCucumberTest.kt | 2 +- src/test/kotlin/ViewTest.kt | 68 +++++++++++++++ src/test/resources/feature/articles.feature | 1 + 17 files changed, 325 insertions(+), 13 deletions(-) create mode 100644 src/main/kotlin/fr/dcproject/entity/ViewAggregation.kt create mode 100644 src/main/kotlin/fr/dcproject/entity/Viewable.kt create mode 100644 src/main/kotlin/fr/dcproject/utils/DateTime.kt create mode 100644 src/main/kotlin/fr/dcproject/utils/Elastic.kt create mode 100644 src/main/kotlin/fr/dcproject/views/ArticleViewManager.kt create mode 100644 src/main/kotlin/fr/dcproject/views/ViewManager.kt create mode 100644 src/test/kotlin/ViewTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 6e3ad7c..e71f15a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -90,6 +90,8 @@ dependencies { implementation("com.sendgrid:sendgrid-java:4.4.1") implementation("io.lettuce:lettuce-core:5.2.2.RELEASE") implementation("com.rabbitmq:amqp-client:5.8.0") + implementation("org.elasticsearch.client:elasticsearch-rest-client:6.7.1") + implementation("com.jayway.jsonpath:json-path:2.4.0") testImplementation("io.ktor:ktor-server-tests:$ktor_version") testImplementation("io.ktor:ktor-client-mock:$ktor_version") diff --git a/src/main/kotlin/fr/dcproject/Application.kt b/src/main/kotlin/fr/dcproject/Application.kt index c13f748..d15fcdb 100644 --- a/src/main/kotlin/fr/dcproject/Application.kt +++ b/src/main/kotlin/fr/dcproject/Application.kt @@ -36,6 +36,8 @@ import io.ktor.routing.Routing import io.ktor.util.KtorExperimentalAPI import io.ktor.websocket.WebSockets import org.eclipse.jetty.util.log.Slf4jLog +import org.elasticsearch.client.Request +import org.elasticsearch.client.RestClient import org.koin.core.qualifier.named import org.koin.ktor.ext.Koin import org.koin.ktor.ext.get @@ -169,6 +171,57 @@ fun Application.module(env: Env = PROD) { } } + /* Create index if not exist */ + get().run { + if (performRequest(Request("HEAD", "/views?include_type_name=false")).statusLine.statusCode == 404) { + Request( + "PUT", + "/views?include_type_name=false" + ).apply { + //language=JSON + setJsonEntity( + """ + { + "settings": { + "number_of_shards": 5 + }, + "mappings": { + "properties": { + "logged": { + "type": "boolean" + }, + "type": { + "type": "keyword" + }, + "user_ref": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "version_id": { + "type": "keyword" + }, + "ip": { + "type": "keyword" + }, + "citizen_id": { + "type": "keyword" + }, + "view_at": { + "type": "date" + } + } + } + } + """.trimIndent() + ) + }.let { + performRequest(it) + } + } + } + install(WebSockets) { pingPeriod = Duration.ofSeconds(60) // Disabled (null) by default timeout = Duration.ofSeconds(15) @@ -233,7 +286,7 @@ fun Application.module(env: Env = PROD) { install(Routing) { // trace { application.log.trace(it.buildText()) } authenticate(optional = true) { - article(get()) + article(get(), get()) auth(get(), get(), get()) citizen(get(), get()) constitution(get()) diff --git a/src/main/kotlin/fr/dcproject/ApplicationContext.kt b/src/main/kotlin/fr/dcproject/ApplicationContext.kt index 349ce2d..f90cfe3 100644 --- a/src/main/kotlin/fr/dcproject/ApplicationContext.kt +++ b/src/main/kotlin/fr/dcproject/ApplicationContext.kt @@ -6,7 +6,6 @@ import io.ktor.application.ApplicationCall import io.ktor.auth.authentication import io.ktor.util.AttributeKey import io.ktor.util.pipeline.PipelineContext -import kotlinx.coroutines.runBlocking import org.koin.core.context.GlobalContext import fr.dcproject.entity.Citizen as CitizenEntity import fr.dcproject.repository.Citizen as CitizenRepository @@ -15,11 +14,15 @@ private val citizenAttributeKey = AttributeKey("CitizenContext") val ApplicationCall.citizen: CitizenEntity get() = attributes.computeIfAbsent(citizenAttributeKey) { - runBlocking { - val user = authentication.principal() ?: throw ForbiddenException() - GlobalContext.get().koin.get().findByUser(user) - ?: throw ForbiddenException("Citizen not found for this user id \"${user.id}\"") - } + val user = authentication.principal() ?: throw ForbiddenException() + GlobalContext.get().koin.get().findByUser(user) + ?: throw ForbiddenException("Citizen not found for this user id \"${user.id}\"") + } + +val ApplicationCall.citizenOrNull: CitizenEntity? + get() = authentication.principal()?.let { + GlobalContext.get().koin.get().findByUser(it) } val PipelineContext.citizen get() = context.citizen +val PipelineContext.citizenOrNull get() = context.citizenOrNull diff --git a/src/main/kotlin/fr/dcproject/Module.kt b/src/main/kotlin/fr/dcproject/Module.kt index 6f30cf5..995b659 100644 --- a/src/main/kotlin/fr/dcproject/Module.kt +++ b/src/main/kotlin/fr/dcproject/Module.kt @@ -10,6 +10,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.rabbitmq.client.ConnectionFactory import fr.dcproject.messages.Mailer import fr.dcproject.messages.SsoManager +import fr.dcproject.views.ArticleViewManager import fr.postgresjson.connexion.Connection import fr.postgresjson.connexion.Requester import fr.postgresjson.migration.Migrations @@ -18,6 +19,8 @@ import io.ktor.client.features.websocket.WebSockets import io.ktor.util.KtorExperimentalAPI import io.lettuce.core.RedisClient import io.lettuce.core.api.async.RedisAsyncCommands +import org.apache.http.HttpHost +import org.elasticsearch.client.RestClient import org.koin.core.qualifier.named import org.koin.dsl.module import fr.dcproject.repository.Article as ArticleRepository @@ -109,6 +112,15 @@ val Module = module { single { OpinionChoiceRepository(get()) } single { OpinionArticleRepository(get()) } + // Elasticsearch Client + single { + RestClient.builder( + HttpHost("localhost", 9200, "http") + ).build() + } + + single { ArticleViewManager(get()) } + // Mailler single { Mailer(config.sendGridKey) } diff --git a/src/main/kotlin/fr/dcproject/entity/Article.kt b/src/main/kotlin/fr/dcproject/entity/Article.kt index 7d34277..dda85fe 100644 --- a/src/main/kotlin/fr/dcproject/entity/Article.kt +++ b/src/main/kotlin/fr/dcproject/entity/Article.kt @@ -18,7 +18,8 @@ class Article( override var lastVersion: Boolean = false, createdBy: CitizenBasic ) : ArticleFull, - ArticleBasic(id, title, anonymous, content, description, tags, createdBy) + ArticleBasic(id, title, anonymous, content, description, tags, createdBy), + Viewable by ViewableImp() open class ArticleBasic( id: UUID = UUID.randomUUID(), @@ -41,14 +42,20 @@ open class ArticleSimple( override var title: String, override val createdBy: CitizenBasic ) : ArticleSimpleI, - ArticleRef(id), + ArticleRefVersioning(id), EntityCreatedAt by EntityCreatedAtImp(), EntityCreatedBy by EntityCreatedByImp(createdBy), EntityDeletedAt by EntityDeletedAtImp(), - EntityVersioning by UuidEntityVersioning(), Votable by VotableImp(), Opinionable by OpinionableImp() +open class ArticleRefVersioning( + id: UUID = UUID.randomUUID(), + versionNumber: Int? = null, + versionId: UUID = UUID.randomUUID() +) : ArticleRef(id), + EntityVersioning by UuidEntityVersioning(versionNumber, versionId) + open class ArticleRef( id: UUID = UUID.randomUUID() ) : ArticleI, TargetRef(id) diff --git a/src/main/kotlin/fr/dcproject/entity/ViewAggregation.kt b/src/main/kotlin/fr/dcproject/entity/ViewAggregation.kt new file mode 100644 index 0000000..9b810e7 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/entity/ViewAggregation.kt @@ -0,0 +1,13 @@ +package fr.dcproject.entity + +import fr.postgresjson.entity.EntityI +import fr.postgresjson.entity.immutable.EntityUpdatedAt +import fr.postgresjson.entity.immutable.EntityUpdatedAtImp + +open class ViewAggregation( + val total: Int, + val unique: Int +) : EntityI, + EntityUpdatedAt by EntityUpdatedAtImp() { + constructor() : this(0, 0) +} diff --git a/src/main/kotlin/fr/dcproject/entity/Viewable.kt b/src/main/kotlin/fr/dcproject/entity/Viewable.kt new file mode 100644 index 0000000..ddbb8b0 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/entity/Viewable.kt @@ -0,0 +1,9 @@ +package fr.dcproject.entity + +interface Viewable { + var views: ViewAggregation +} + +class ViewableImp : Viewable { + override var views: ViewAggregation = ViewAggregation() +} \ No newline at end of file diff --git a/src/main/kotlin/fr/dcproject/event/EventSubscriber.kt b/src/main/kotlin/fr/dcproject/event/EventSubscriber.kt index 0661b42..0abd3f8 100644 --- a/src/main/kotlin/fr/dcproject/event/EventSubscriber.kt +++ b/src/main/kotlin/fr/dcproject/event/EventSubscriber.kt @@ -20,7 +20,7 @@ abstract class Event( } } -abstract class EntityEvent( +open class EntityEvent( val target: UuidEntity, type: String, val action: String diff --git a/src/main/kotlin/fr/dcproject/routes/Article.kt b/src/main/kotlin/fr/dcproject/routes/Article.kt index b76ffa9..21001ce 100644 --- a/src/main/kotlin/fr/dcproject/routes/Article.kt +++ b/src/main/kotlin/fr/dcproject/routes/Article.kt @@ -1,11 +1,13 @@ package fr.dcproject.routes import fr.dcproject.citizen +import fr.dcproject.citizenOrNull import fr.dcproject.event.ArticleUpdate import fr.dcproject.repository.Article.Filter import fr.dcproject.security.voter.ArticleVoter.Action.CREATE import fr.dcproject.security.voter.ArticleVoter.Action.VIEW import fr.dcproject.security.voter.assertCan +import fr.dcproject.views.ArticleViewManager import fr.postgresjson.repository.RepositoryI import io.ktor.application.application import io.ktor.application.call @@ -16,6 +18,7 @@ import io.ktor.locations.post import io.ktor.request.receive import io.ktor.response.respond import io.ktor.routing.Route +import kotlinx.coroutines.launch import fr.dcproject.entity.Article as ArticleEntity import fr.dcproject.entity.request.Article as ArticleEntityRequest import fr.dcproject.repository.Article as ArticleRepository @@ -56,7 +59,7 @@ object ArticlesPaths { } @KtorExperimentalLocationsAPI -fun Route.article(repo: ArticleRepository) { +fun Route.article(repo: ArticleRepository, viewManager: ArticleViewManager) { get { val articles = repo.find(it.page, it.limit, it.sort, it.direction, it.search, Filter(createdById = it.createdBy)) @@ -67,7 +70,13 @@ fun Route.article(repo: ArticleRepository) { get { assertCan(VIEW, it.article) + it.article.views = viewManager.getViewsCount(it.article) + call.respond(it.article) + + launch { + viewManager.addView(call.request.local.remoteHost, it.article, citizenOrNull) + } } get { diff --git a/src/main/kotlin/fr/dcproject/utils/DateTime.kt b/src/main/kotlin/fr/dcproject/utils/DateTime.kt new file mode 100644 index 0000000..b720614 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/utils/DateTime.kt @@ -0,0 +1,6 @@ +package fr.dcproject.utils + +import org.joda.time.DateTime +import org.joda.time.format.ISODateTimeFormat + +fun DateTime.toIso() = ISODateTimeFormat.dateTime().print(this) \ No newline at end of file diff --git a/src/main/kotlin/fr/dcproject/utils/Elastic.kt b/src/main/kotlin/fr/dcproject/utils/Elastic.kt new file mode 100644 index 0000000..3962a2c --- /dev/null +++ b/src/main/kotlin/fr/dcproject/utils/Elastic.kt @@ -0,0 +1,29 @@ +package fr.dcproject.utils + +import com.jayway.jsonpath.JsonPath +import com.jayway.jsonpath.PathNotFoundException +import org.apache.http.util.EntityUtils +import org.elasticsearch.client.Response +import org.slf4j.LoggerFactory + +fun Response.contentToString(): String { + return EntityUtils.toString(this.entity) +} + +fun Response.getField(jsonPath: String): Int? { + return try { + JsonPath.read(this.contentToString(), jsonPath) + } catch (e: PathNotFoundException) { + null + } +} + +fun String.getJsonField(jsonPath: String): Int? { + return try { + JsonPath.read(this, jsonPath) + } catch (e: PathNotFoundException) { + LoggerFactory.getLogger("fr.dcproject.utils.getJsonField") + .warn("No value for Json path ${JsonPath.compile(jsonPath).path}") + null + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/dcproject/views/ArticleViewManager.kt b/src/main/kotlin/fr/dcproject/views/ArticleViewManager.kt new file mode 100644 index 0000000..ef5c9e7 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/views/ArticleViewManager.kt @@ -0,0 +1,86 @@ +package fr.dcproject.views + +import fr.dcproject.entity.* +import fr.dcproject.utils.contentToString +import fr.dcproject.utils.getJsonField +import fr.dcproject.utils.toIso +import io.ktor.util.sha1 +import org.elasticsearch.client.Request +import org.elasticsearch.client.Response +import org.elasticsearch.client.RestClient +import org.joda.time.DateTime +import java.util.* + +class ArticleViewManager(private val restClient: RestClient) : ViewManager { + override fun addView(ip: String, article: ArticleRefVersioning, citizen: CitizenRef?, dateTime: DateTime): Response? { + val isLogged = (citizen != null).toString() + sha1("plop".toByteArray()) + val ref = citizen?.id ?: UUID.nameUUIDFromBytes(ip.toByteArray())!! + val request = Request( + "POST", + "/views/_doc/" + ).apply { + //language=JSON + setJsonEntity(""" + { + "logged": $isLogged, + "type": "article", + "user_ref": "$ref", + "ip": "$ip", + "id": "${article.id}", + "version_id": "${article.versionId}", + "citizen_id": "${citizen?.id}", + "view_at": "${dateTime.toIso()}" + } + """.trimIndent()) + } + + return restClient.performRequest(request) + } + + override fun getViewsCount(article: ArticleRefVersioning): ViewAggregation { + val request = Request( + "GET", + "/views/_search" + ).apply { + //language=JSON + setJsonEntity(""" + { + "size": 0, + "query": { + "bool": { + "must": { + "term": { + "version_id": "${article.versionId}" + } + } + } + }, + "aggs" : { + "total": { + "composite" : { + "sources" : [ + { "version_id": { "terms": {"field": "version_id" } } } + ] + } + }, + "unique" : { + "cardinality" : { + "field" : "user_ref", + "precision_threshold": 1 + } + } + } + } + """.trimIndent()) + } + + return restClient + .performRequest(request).contentToString().run { + ViewAggregation( + getJsonField("$.aggregations.total.buckets[0].doc_count") ?: 0, + getJsonField("$.aggregations.unique.value") ?: 0 + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/dcproject/views/ViewManager.kt b/src/main/kotlin/fr/dcproject/views/ViewManager.kt new file mode 100644 index 0000000..95a0887 --- /dev/null +++ b/src/main/kotlin/fr/dcproject/views/ViewManager.kt @@ -0,0 +1,11 @@ +package fr.dcproject.views + +import fr.dcproject.entity.CitizenRef +import fr.dcproject.entity.ViewAggregation +import org.elasticsearch.client.Response +import org.joda.time.DateTime + +interface ViewManager { + fun addView(ip: String, entity: T, citizen: CitizenRef? = null, dateTime: DateTime = DateTime.now()): Response? + fun getViewsCount(entity: T): ViewAggregation +} \ No newline at end of file diff --git a/src/main/resources/openapi.yaml b/src/main/resources/openapi.yaml index cbc81a5..c9e3cb4 100644 --- a/src/main/resources/openapi.yaml +++ b/src/main/resources/openapi.yaml @@ -258,6 +258,9 @@ paths: parameters: - $ref: '#/components/parameters/article' get: + security: + - {} + - JWTAuth: [] summary: Get one article tags: - article diff --git a/src/test/kotlin/RunCucumberTest.kt b/src/test/kotlin/RunCucumberTest.kt index 79f3e0a..ccb2f1e 100644 --- a/src/test/kotlin/RunCucumberTest.kt +++ b/src/test/kotlin/RunCucumberTest.kt @@ -25,7 +25,7 @@ var unitialized: Boolean = false @KtorExperimentalAPI @KtorExperimentalLocationsAPI @RunWith(Cucumber::class) -@CucumberOptions(plugin = ["pretty"]) +@CucumberOptions(plugin = ["pretty"], strict = true) class RunCucumberTest : En, KoinTest { private val logger: Logger? by LoggerDelegate() diff --git a/src/test/kotlin/ViewTest.kt b/src/test/kotlin/ViewTest.kt new file mode 100644 index 0000000..6e903e6 --- /dev/null +++ b/src/test/kotlin/ViewTest.kt @@ -0,0 +1,68 @@ +import fr.dcproject.Env +import fr.dcproject.entity.ArticleRefVersioning +import fr.dcproject.entity.CitizenRef +import fr.dcproject.module +import fr.dcproject.views.ArticleViewManager +import io.ktor.locations.KtorExperimentalLocationsAPI +import io.ktor.server.testing.withTestApplication +import io.ktor.util.KtorExperimentalAPI +import org.amshove.kluent.`should equal` +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS +import org.koin.ktor.ext.get +import java.util.* + +@KtorExperimentalLocationsAPI +@KtorExperimentalAPI +@TestInstance(PER_CLASS) +class ViewTest { + @Test + fun `test View Article`() { + val article = ArticleRefVersioning(id = UUID.randomUUID(), versionId = UUID.randomUUID()) + val citizenRef = CitizenRef() + + withTestApplication({ module(Env.TEST) }) { + val viewManager: ArticleViewManager = application.get() + + /* Get view before */ + val startView = viewManager.getViewsCount(article) + + /* Add View */ + viewManager.addView( + "1.2.3.4", + article, + citizenRef + ) + + /* Add View */ + viewManager.addView( + "10.10.10.10", + article, + citizenRef + ) + + /* Add View */ + viewManager.addView( + "8.8.8.8", + article + ) + + /* Add View */ + viewManager.addView( + "1.1.1.1", + article + ) + + /* Sleep because ES is not sync ! */ + Thread.sleep(1000) + + /* Get view */ + val afterView = viewManager.getViewsCount(article) + + /* Check if view has increment */ + afterView.total `should equal` startView.total + 4 + afterView.unique `should equal` startView.unique + 3 + } + } +} diff --git a/src/test/resources/feature/articles.feature b/src/test/resources/feature/articles.feature index 5bdd35d..f18fcd0 100644 --- a/src/test/resources/feature/articles.feature +++ b/src/test/resources/feature/articles.feature @@ -1,3 +1,4 @@ +@article Feature: articles routes Scenario: The route for get articles must response a 200