Allow setting the in-reply-to header

rtyley · rtyley · commit b23ac9b2dd91 · 2015-06-30T18:45:31.000+01:00
Typeahead for identifying message-ids in a human-readable way Particularly like linking back to the Gmane article... lotsa jquery Add bootstrap-3-specific styling to the typeahead.js stuff Note the workaround for `typeahead.js-bootstrap3.less` not having quite updated to cope with a class-name change in `typeahead.js` v0.11.1 (made the suggestions background-transparent) - just tell typeahead.js to use the old class name: hyspace/typeahead.js-bootstrap3.less#24 (comment)
diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala
@@ -10,17 +10,17 @@ import lib.actions.Actions._
 import lib.actions.Requests._
 import lib.aws.SES._
 import lib.aws.SesAsyncHelpers._
-import lib.model.{PatchBomb, PatchCommit, Patch}
+import lib.model.PatchBomb
 import org.eclipse.jgit.lib.ObjectId
 import org.kohsuke.github._
 import play.api.Logger
 import play.api.data.Form
 import play.api.data.Forms._
+import play.api.i18n.Messages.Implicits._
 import play.api.libs.json.Json
 import play.api.libs.json.Json._
 import play.api.mvc._
 import views.html.pullRequestSent
-import play.api.i18n.Messages.Implicits._
 
 import scala.collection.convert.wrapAll._
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -85,7 +85,8 @@ object Application extends Controller {
 
   val mailSettingsForm = Form(
     mapping(
-      "subjectPrefix" -> default(text(maxLength = 20), "PATCH")
+      "subjectPrefix" -> default(text(maxLength = 20), "PATCH"),
+      "inReplyTo" -> optional(text)
     )(PRMailSettings.apply)(PRMailSettings.unapply)
   )
 
@@ -99,17 +100,26 @@ object Application extends Controller {
       
       for (patchCommits <- req.patchCommitsF) yield {
         val patchBomb = PatchBomb(patchCommits, addresses, settings.subjectPrefix, mailType.subjectPrefix, mailType.footer(req.pr))
-        for (initialMessageId <- ses.send(patchBomb.emails.head)) {
+        val initialEmail = patchBomb.emails.head
+        val initialEmailWithReply = settings.inReplyTo.fold(initialEmail)(initialEmail.inReplyTo)
+        for (initialMessageId <- ses.send(initialEmailWithReply)) {
           for (email <- patchBomb.emails.drop(1)) {
             ses.send(email.inReplyTo(initialMessageId))
           }
 
           mailType.afterSending(req.pr, initialMessageId)
         }
-        Ok(pullRequestSent(req.pr, req.user, mailType)).addingToSession(prId.slug -> Json.toJson(settings).toString)
+        Ok(pullRequestSent(req.pr, req.user, mailType)).addingToSession(prId.slug -> toJson(settings).toString)
       }
   }
 
+  def messageLookup(repoId: RepoId, query: String) = Action.async {
+    val archives = Project.byRepoId(repoId).mailingList.archives
+    for {
+      messagesOpt <- Future.find(archives.map(_.lookupMessage(query)))(_.nonEmpty)
+    } yield Ok(toJson(messagesOpt.toSeq.flatten: Seq[MessageSummary]))
+  }
+
   lazy val gitCommitId = {
     val g = gitCommitIdFromHerokuFile
     Logger.info(s"Heroku dyno commit id $g")
diff --git a/app/lib/MailArchive.scala b/app/lib/MailArchive.scala
@@ -1,11 +1,67 @@
 package lib
 
+import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder}
+import java.time.temporal.ChronoField._
+import java.time.{LocalDateTime, ZoneId, ZonedDateTime}
+import javax.mail.internet.MailDateFormat
+
+import com.madgag.okhttpscala._
+import com.squareup.okhttp
+import com.squareup.okhttp.OkHttpClient
+import controllers.Application._
+import fastparse.core.Result
+import lib.Email.Addresses
+import lib.model.PatchParsing
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+import play.api.libs.json.Json
+
+import scala.collection.convert.wrapAsScala._
+import scala.concurrent.{ExecutionContext, Future}
+
+case class MessageSummary(id: String, subject: String, date: ZonedDateTime, addresses: Addresses, groupLink: String)
+
+object MessageSummary {
+  implicit val formatsAddresses = Json.format[Addresses]
+  implicit val formatsMessageSummary = Json.format[MessageSummary]
+
+  def fromRawMessage(rawMessage: String, articleUrl: String): MessageSummary = {
+    val Result.Success(headers, _) = PatchParsing.headers.parse(rawMessage)
+    val headerMap = headers.toMap
+
+    val messageId = headerMap("Message-ID").stripPrefix("<").stripSuffix(">")
+    val from = headerMap("From")
+    val date = new MailDateFormat().parse(headerMap("Date")).toInstant.atZone(ZoneId.of("UTC"))
+    MessageSummary(messageId, headerMap("Subject"), date, Addresses(from), articleUrl)
+  }
+}
+
+object RedirectCapturer {
+  val okClient = {
+    val c = new OkHttpClient()
+    c.setFollowRedirects(false)
+    c
+  }
+
+  def redirectFor(url: String)(implicit ec: ExecutionContext): Future[Option[String]] = for {
+    resp <- okClient.execute(new okhttp.Request.Builder().url(url).build())
+  } yield {
+    resp.code match {
+      case FOUND => Some(resp.header(LOCATION))
+      case _ => None
+    }
+  }
+}
+
+
 trait MailArchive {
   val providerName: String
 
   val url: String
 
   def linkFor(messageId: String): String
+
+  def lookupMessage(query: String)(implicit ec: ExecutionContext): Future[Seq[MessageSummary]] = Future.successful(Seq.empty)
 }
 
 case class Gmane(groupName: String) extends MailArchive {
@@ -14,6 +70,82 @@ case class Gmane(groupName: String) extends MailArchive {
   val url = s"http://dir.gmane.org/gmane.$groupName"
 
   def linkFor(messageId: String) = s"http://mid.gmane.org/$messageId"
+
+  override def lookupMessage(query: String)(implicit ec: ExecutionContext) = {
+    for {
+      gmaneArticleUrlOpt <- gmaneArticleUrlFor(query)
+      gmaneRawArticleOpt <- gmaneRawArticleFor(gmaneArticleUrlOpt)
+    } yield gmaneRawArticleOpt.toSeq
+  }
+
+  def gmaneRawArticleFor(articleUrlOpt: Option[String])(implicit ec: ExecutionContext): Future[Option[MessageSummary]] = {
+    articleUrlOpt match {
+      case Some(articleUrl) =>
+        val okClient = new OkHttpClient()
+        for {
+          resp <- okClient.execute(new okhttp.Request.Builder().url(articleUrl+"/raw").build())
+        } yield Some(MessageSummary.fromRawMessage(resp.body.string, articleUrl))
+      case None => Future.successful(None)
+    }
+  }
+
+  def gmaneArticleUrlFor(messageId: String)(implicit ec: ExecutionContext): Future[Option[String]] =
+    RedirectCapturer.redirectFor(linkFor(messageId))
+
+}
+
+object Marc {
+  val Git = Marc("git")
+
+  def deobfuscate(emailOrMessageId: String) = emailOrMessageId.replace(" () ","@").replace(" ! ", ".")
+
+  /*
+   * Hacks EVERYWHERE - MARC unfortunately doesn't expose this data in a nice format for us
+   */
+  def messageSummaryFor(articleHtml: String): MessageSummary = {
+    val elements = Jsoup.parse(articleHtml).select("""pre b font[size="+1"]""")
+    val nodes = elements.get(0).childNodes().toList
+    val headerMap = (for { header :: value :: Nil <- nodes.grouped(2) } yield {
+      header.outerHtml.trim.stripSuffix(":") -> value.asInstanceOf[Element].html()
+    }).toMap
+    val messageId = deobfuscate(headerMap("Message-ID"))
+    val from = deobfuscate(headerMap("From"))
+
+    val ISO_LOCAL_TIME = new DateTimeFormatterBuilder().appendValue(HOUR_OF_DAY).appendLiteral(':').appendValue(MINUTE_OF_HOUR).optionalStart.appendLiteral(':').appendValue(SECOND_OF_MINUTE).toFormatter
+    val ISO_LOCAL_DATE_TIME = new DateTimeFormatterBuilder().parseCaseInsensitive.append(DateTimeFormatter.ISO_LOCAL_DATE).appendLiteral('T').append(ISO_LOCAL_TIME).toFormatter
+
+    val date = LocalDateTime.parse(headerMap("Date").replace(' ', 'T'), ISO_LOCAL_DATE_TIME).atZone(ZoneId.of("UTC"))
+    MessageSummary(messageId, headerMap("Subject"), date, Addresses(from), "")
+  }
+}
+
+case class Marc(groupName: String) extends MailArchive {
+  val providerName = "MARC"
+
+  val url = s"http://marc.info/?l=$groupName"
+
+  def linkFor(messageId: String) = s"http://marc.info/?i=$messageId"
+
+  override def lookupMessage(query: String)(implicit ec: ExecutionContext) = {
+    for {
+      articleUrlOpt <- articleUrl(query)
+      articleMessageSummaryOpt <- messageSummaryFor(articleUrlOpt)
+    } yield articleMessageSummaryOpt.map(_.copy(id = query)).toSeq
+  }
+
+  def articleUrl(messageId: String)(implicit ec: ExecutionContext): Future[Option[String]] =
+    RedirectCapturer.redirectFor(linkFor(messageId))
+
+  def messageSummaryFor(articleUrlOpt: Option[String])(implicit ec: ExecutionContext): Future[Option[MessageSummary]] = {
+    articleUrlOpt match {
+      case Some(articleUrl) =>
+        val okClient = new OkHttpClient()
+        for {
+          resp <- okClient.execute(new okhttp.Request.Builder().url(articleUrl).build())
+        } yield Some(Marc.messageSummaryFor(resp.body.string).copy(groupLink = articleUrl))
+      case None => Future.successful(None)
+    }
+  }
 }
 
 object Gmane {
diff --git a/app/lib/PRMailSettings.scala b/app/lib/PRMailSettings.scala
@@ -2,7 +2,7 @@ package lib
 
 import play.api.libs.json.Json
 
-case class PRMailSettings(subjectPrefix: String)
+case class PRMailSettings(subjectPrefix: String, inReplyTo: Option[String] = None)
 
 object PRMailSettings {
   implicit val formats = Json.format[PRMailSettings]
diff --git a/app/lib/ProjectRepo.scala b/app/lib/ProjectRepo.scala
@@ -8,7 +8,8 @@ object Project {
   val Git = Project(
     RepoId("git", "git"),
     MailingList("git@vger.kernel.org", Seq(
-      Gmane.Git
+      Gmane.Git,
+      Marc.Git
       // MailArchiveDotCom("git@vger.kernel.org") -- message-id search appears broken
     ))
   )
diff --git a/app/lib/model/Patch.scala b/app/lib/model/Patch.scala
@@ -19,6 +19,14 @@ object PatchParsing {
 
   val patchHeaderRegion: P[ObjectId] = P(patchFromHeader ~! nonEmptyLine.rep)
 
+  val headerKey = P(CharsWhile(_ != ':', min = 1).! ~ ": ")
+
+  val headerValue = P((CharsWhile(_ != '\n', min = 1) ~ ("\n" ~ " ".rep(min = 1) ~ CharsWhile(_ != '\n', min = 1)).rep).!)
+
+  val header: P[(String, String)] = P(headerKey ~ headerValue)
+
+  val headers: P[Seq[(String, String)]] = P(header.rep(sep = "\n"))
+
   val patchBodyRegion = P((line ~ !patchFromHeader).rep.!)
 
   val patch: P[Patch] = P(patchHeaderRegion ~ "\n" ~! patchBodyRegion).map(Patch.tupled)
diff --git a/app/views/fragments/emailAddresses.scala.html b/app/views/fragments/emailAddresses.scala.html
@@ -1,17 +1,15 @@
-@(addresses: lib.Email.Addresses)
-@display(label: String, ids: Seq[String]) = {
+@(addresses: lib.Email.Addresses)(implicit fc : views.html.b3.B3FieldConstructor)
+
+
+    @display(label: String, ids: Seq[String]) = {
     @if(ids.nonEmpty) {
-        <label class="col-sm-4 control-label">@label</label>
-        <div class="col-sm-8">
-            <p class="form-control-static">@ids.mkString(", ")</p>
-        </div>
+        @b3.static(label) { @ids.mkString(", ")}
+
     }
 }
 
-<div class="form-group">
     @display("From", Seq(addresses.from))
     @display("To", addresses.to)
     @display("Cc", addresses.cc)
     @display("Bcc", addresses.bcc)
     @display("Reply-To", addresses.replyTo.toSeq)
-</div>
diff --git a/app/views/fragments/head.scala.html b/app/views/fragments/head.scala.html
@@ -4,7 +4,12 @@
     <link rel="stylesheet" media="screen" href="@routes.Assets.at("lib/bootstrap/css/bootstrap.min.css")" />
     <script type="text/javascript" src="@routes.Assets.at("lib/jquery/jquery.js")"></script>
     <script type="text/javascript" src="@routes.Assets.at("lib/bootstrap/js/bootstrap.min.js")"></script>
+    <script type="text/javascript" src="@routes.Assets.at("lib/typeahead.js/dist/typeahead.bundle.js")"></script>
+    <script type="text/javascript" src="@routes.Assets.at("lib/timeago/jquery.timeago.js")"></script>
+    <script type="text/javascript" src="@routes.Assets.at("lib/handlebars/dist/handlebars.js")"></script>
+    <script type="text/javascript" src="@routes.Assets.at("javascript/messageIdPicker.js")"></script>
     <link rel="stylesheet" href="@routes.Assets.at("lib/octicons/octicons/octicons.css")">
+    <link rel="stylesheet" href="@routes.Assets.at("lib/typeahead.js-bootstrap3.less/typeahead.css")">
     <link rel="stylesheet" media="screen" href="@routes.Assets.at("stylesheets/main.css")">
 
     @for(googleAnalyticsId <- lib.AnalyticsConfig.googleAnalyticsIdOpt) {
diff --git a/app/views/fragments/mailIdent.scala.html b/app/views/fragments/mailIdent.scala.html
@@ -0,0 +1,7 @@
+@(messageSummary: lib.MessageSummary)
+<small>
+    <h5 title="@messageSummary.subject"
+    style="white-space: nowrap; overflow: hidden; text-overflow: ellipsis; font-weight: bold; margin: 0px;">@messageSummary.subject</h5>
+@messageSummary.addresses.from -
+    <a target="_blank" href="{{groupLink}}"><time class="timeago" datetime="{{date}}">{{date}}</time></a>
+</small>
diff --git a/app/views/fragments/sendTabContent.scala.html b/app/views/fragments/sendTabContent.scala.html
@@ -1,11 +1,12 @@
+@import java.time.Instant
+@import lib.MessageSummary
+@import lib.Email.Addresses
 @(mailType: lib.MailType, proposedMail: lib.ProposedMail, isDefault: Boolean, btnClass: String
         )(implicit req: GHPRRequest[_], prMailSettings: Form[lib.PRMailSettings], messages: Messages)
 @import helper._
-@implicitFieldConstructor = @{ b3.horizontal.fieldConstructor("col-md-4", "col-md-8") }
+@implicitFieldConstructor = @{ b3.vertical.fieldConstructor }
 
 <div role="tabpanel" class="tab-pane @if(isDefault) { fade in active }" id="mail-@mailType.slug">
-@b3.form(routes.Application.mailPullRequest(req.pr.id, mailType)) {
-    @CSRF.formField
 
     @if(proposedMail.errors.nonEmpty) {
         <div class="alert alert-warning" role="alert">
@@ -17,7 +18,12 @@
         </div>
     }
 
-    @fragments.emailAddresses(proposedMail.addresses)
+    <div class="form-horizontal addresses">
+    @fragments.emailAddresses(proposedMail.addresses)(b3.horizontal.fieldConstructor("col-md-2", "col-md-10"))
+    </div>
+
+    @b3.form(routes.Application.mailPullRequest(req.pr.id, mailType)) {
+        @CSRF.formField
 
     @b3.inputWrapped("text", prMailSettings("subjectPrefix"), '_label -> "Subject prefix", 'placeholder -> "PATCH", 'maxlength -> "20") { input =>
         <div class="input-group">
@@ -27,14 +33,49 @@
         </div>
     }
 
-    <div class="form-group">
-        <div class="col-sm-offset-4 col-sm-8">
-            <button
-            class="btn @btnClass btn-lg"
-            value="send"
-                @if(proposedMail.errors.nonEmpty) { disabled="disabled" }
-            >Send</button>
+    @b3.inputWrapped("text", prMailSettings("inReplyTo"), '_label -> "In-reply-to", 'placeholder -> "message-id (optional)", 'class -> "form-control typeahead") { input =>
+        <div id="@mailType.slug-in-reply-to">
+            <div class="input-group">
+                <span class="input-group-addon">&lt;</span>
+                @input
+                <span class="input-group-addon">&gt;</span>
+            </div>
+            <div class="message-ident panel panel-default" style="padding: 3%; display: none;">
+                <button type="button" class="close" aria-label="Remove"><span aria-hidden="true">&times;</span></button>
+                <div class="message-ident-content"></div>
+            </div>
         </div>
-    </div>
+    }
+
+    <script id="entry-template" type="text/x-handlebars-template">
+        @fragments.mailIdent(MessageSummary("{{id}}", "{{subject}}", java.time.ZonedDateTime.now, Addresses("{{addresses.from}}"), "{{groupLink}}"))
+    </script>
+    <script>
+        jQuery("#@mailType.slug-in-reply-to").messageIdPicker({
+            hint: true,
+            highlight: true,
+            minLength: 3,
+            classNames: {menu: 'tt-dropdown-menu'}
+        }, {
+            name: 'messageIds',
+            source: new Bloodhound({
+                datumTokenizer: Bloodhound.tokenizers.obj.whitespace('value'),
+                queryTokenizer: Bloodhound.tokenizers.whitespace,
+                remote: {
+                    url: '@routes.Application.messageLookup(req.repo.id, "QUERY_STRING")',
+                    wildcard: 'QUERY_STRING'
+                }
+            }),
+            display: 'id',
+            templates: {
+                empty: '<div class="empty-message">Unable to find any Messages that match that Id</div>',
+                suggestion: Handlebars.compile(jQuery("#entry-template").html())
+            }
+        });
+    </script>
+
+    @b3.submit('class -> s"btn $btnClass btn-lg", 'disabled -> proposedMail.errors.nonEmpty) {
+        Send
+    }
 }
 </div>
diff --git a/build.sbt b/build.sbt
@@ -38,7 +38,12 @@ libraryDependencies ++= Seq(
   "org.webjars" % "bootstrap" % "3.3.4",
   "com.adrianhurt" %% "play-bootstrap3" % "0.4.4-P24",
   "org.webjars.bower" % "octicons" % "2.2.3",
+  "org.webjars.bower" % "timeago" % "1.4.1",
+  "org.webjars.bower" % "typeahead.js" % "0.11.1",
+  "org.webjars.bower" % "typeahead.js-bootstrap3.less" % "0.2.3",
+  "org.webjars.npm" % "handlebars" % "3.0.3",
   "com.lihaoyi" %% "fastparse" % "0.1.7",
+  "org.jsoup" % "jsoup" % "1.8.2",
   "com.github.nscala-time" %% "nscala-time" % "2.0.0",
   "com.github.scala-incubator.io" %% "scala-io-file" % "0.4.3-1",
   "org.specs2" %% "specs2-core" % "2.4.17" % "test",
diff --git a/conf/routes b/conf/routes
diff --git a/public/javascript/messageIdPicker.js b/public/javascript/messageIdPicker.js
diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css
diff --git a/test/lib/MailArchiveSpec.scala b/test/lib/MailArchiveSpec.scala
diff --git a/test/resources/samples/mailarchives/marc/m.143228360912708.html b/test/resources/samples/mailarchives/marc/m.143228360912708.html
