Applying templating to our Golang web app

Hello and welcome to part 17 of the Golang tutorial series. We just covered templating with the html/template package, and now we're ready to apply it to our news aggregator.

Full code from the previous tutorial:

package main

import (
	"fmt"
	"net/http"
	"html/template"
)

type NewsAggPage struct {
    Title string
    News string
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "<h1>Whoa, Go is neat!</h1>")
}

func newsAggHandler(w http.ResponseWriter, r *http.Request) {
	p := NewsAggPage{Title: "Amazing News Aggregator", News: "some news"}
    t, _ := template.ParseFiles("basictemplating.html")
    t.Execute(w, p)
}

func main() {
	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/agg/", newsAggHandler)
	http.ListenAndServe(":8000", nil) 
}

Our full code from the latest news aggregator application is:

package main

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Sitemapindex struct {
	Locations []string `xml:"sitemap>loc"`
}

type News struct {
	Titles []string `xml:"url>news>title"`
	Keywords []string `xml:"url>news>keywords"`
	Locations []string `xml:"url>loc"`
}

type NewsMap struct {
	Keyword string
	Location string
}

func main() {
	var s Sitemapindex
	var n News
	resp, _ := http.Get("https://www.washingtonpost.com/news-sitemap-index.xml")
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &s)
	news_map := make(map[string]NewsMap)

	for _, Location := range s.Locations {
		resp, _ := http.Get(Location)
		bytes, _ := ioutil.ReadAll(resp.Body)
		xml.Unmarshal(bytes, &n)

		for idx, _ := range n.Keywords {
			news_map[n.Titles[idx]] = NewsMap{n.Keywords[idx], n.Locations[idx]}
		}
	}
	for idx, data := range news_map {
		fmt.Println("\n\n\n\n\n",idx)
		fmt.Println("\n",data.Keyword)
		fmt.Println("\n",data.Location)
	}
}

We basically want to combine these two programs at this point, and modify our template a bit.

The first thing we're going to is modify the NewsAggPage type. We were initially using the News value as a string, but now we want this to be a map:

type NewsAggPage struct {
    Title string
    News map[string]NewsMap
}

Then we'll add the encoding/xml and io/ioutil imports:

import (
	"fmt"
	"net/http"
	"html/template"
	"encoding/xml"
	"io/ioutil"
)

Next, let's take the NewsMap, Sitemapindex, and News definitions from our latest news aggregator app and add those into our webapp, so now all of our structs:

type NewsMap struct {
	Keyword string
	Location string
}

type NewsAggPage struct {
    Title string
    News map[string]NewsMap
}

type Sitemapindex struct {
	Locations []string `xml:"sitemap>loc"`
}

type News struct {
	Titles []string `xml:"url>news>title"`
	Keywords []string `xml:"url>news>keywords"`
	Locations []string `xml:"url>loc"`
}

Next, we need to work on the newsAggHandler. We just need to take the following code from our latest news aggregator program:

	var s Sitemapindex
	var n News
	resp, _ := http.Get("https://www.washingtonpost.com/news-sitemap-index.xml")
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &s)
	news_map := make(map[string]NewsMap)

	for _, Location := range s.Locations {
		resp, _ := http.Get(Location)
		bytes, _ := ioutil.ReadAll(resp.Body)
		xml.Unmarshal(bytes, &n)

		for idx, _ := range n.Keywords {
			news_map[n.Titles[idx]] = NewsMap{n.Keywords[idx], n.Locations[idx]}
		}

and put this in the web app's newsAggHandler, and then modify the line for defining p to:

p := NewsAggPage{Title: "Amazing News Aggregator", News: news_map}

Then we'll change the template.ParseFiles to newsaggtemplate.html

    t, _ := template.ParseFiles("newsaggtemplate.html")

Now, our full webapp code:

package main

import (
	"fmt"
	"net/http"
	"html/template"
	"encoding/xml"
	"io/ioutil"
)

type NewsMap struct {
	Keyword string
	Location string
}

type NewsAggPage struct {
    Title string
    News map[string]NewsMap
}

type Sitemapindex struct {
	Locations []string `xml:"sitemap>loc"`
}

type News struct {
	Titles []string `xml:"url>news>title"`
	Keywords []string `xml:"url>news>keywords"`
	Locations []string `xml:"url>loc"`
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "<h1>Whoa, Go is neat!</h1>")
}

func newsAggHandler(w http.ResponseWriter, r *http.Request) {
	var s Sitemapindex
	var n News
	resp, _ := http.Get("https://www.washingtonpost.com/news-sitemap-index.xml")
	bytes, _ := ioutil.ReadAll(resp.Body)
	xml.Unmarshal(bytes, &s)
	news_map := make(map[string]NewsMap)

	for _, Location := range s.Locations {
		resp, _ := http.Get(Location)
		bytes, _ := ioutil.ReadAll(resp.Body)
		xml.Unmarshal(bytes, &n)

		for idx, _ := range n.Keywords {
			news_map[n.Titles[idx]] = NewsMap{n.Keywords[idx], n.Locations[idx]}
		}
	}

	p := NewsAggPage{Title: "Amazing News Aggregator", News: news_map}
    t, _ := template.ParseFiles("newsaggtemplate.html")
    t.Execute(w, p)
}

func main() {
	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/agg/", newsAggHandler)
	http.ListenAndServe(":8000", nil) 
}

Now, we need to work on our template, which we'll call newsaggtemplate.html. What we're going to do is build a table. If you do not know HTML, this is not an HTML tutorial. If you need one, there are many online. To begin, we give show the page title:

<h1>{{.Title}}</h1>

Now we can build our table. A typical table would be something like:

<table>
    <col width="35%">
    <col width="65%">
    <thead>
        <tr>
            <th>Title</th>
            <th>Keywords</th>
        </tr>
    </thead>
    <tbody>
		 <tr>
            <td>some title</td>
            <td>some keywords</td>
        </tr>
        <tr>
            <td>another title</td>
            <td>the other keywords</td>
        </tr>
    </tbody>
</table>

What we then want is to iterate through our NewsMap type, which we've passed to our document in the NewsAggPage type, under the News name

	p := NewsAggPage{Title: "Amazing News Aggregator", News: news_map}
    t, _ := template.ParseFiles("aggregatorfinish.html")
    t.Execute(w, p)

So our template has access to this data under a variable called News, which we can reference with {{ .News }}. Next, we want to iterate over this, rather than just simply output News. To iterate, we will again use range, only the syntax is slightly different. In our HTML template, we use range $key, $value := .YourVar. When you are just displaying a single value, you can use {{ .SingleValue }}, but, when you begin a loop, you need some way to let the templating know that the loop is over, so you will also need an ending to it:

        {{ range $key, $value := .News }}
         ...do some stuff
        {{ end }}

In the "do some stuff" part, you can make reference to the key and value. The reason you need to separate the logic like this is so that you can combine the html and the logic. So, for example, let's create a table row with our variables per item in News.

        {{ range $key, $value := .News }}
         <tr>
            <td><a href="{{ $value.Location }}" target='_blank'>{{ $key }}</td>
            <td>{{ $value.Keyword }}</td>
        </tr>
        {{ end }}

Notice how you can use the {{ logic within quotes too. So, here, each row of our table has the title, which links to the article, and then has all of the keywords in the 2nd column.

Our full template code:

<h1>{{.Title}}</h1>

<table>
    <col width="35%">
    <col width="65%">
    <thead>
        <tr>
            <th>Title</th>
            <th>Keywords</th>
        </tr>
    </thead>
    <tbody>
		{{ range $key, $value := .News }}
		 <tr>
            <td><a href="{{ $value.Location }}" target='_blank'>{{ $key }}</td>
            <td>{{ $value.Keyword }}</td>
        </tr>
		{{ end }}
    </tbody>
</table>

At this point, we can run the go webapp file, and visit http://127.0.0.1:8000/agg/ in our browser to see our giant table. One option we have to quickly and easily make this page a bit better is to use something like DataTables, which is a JavaScript plugin that can quickly improve your data-intensive tables. To apply this, we'll need to bring in jquery, the datatables css, and then the datatables javascript. To do this, we'll add the following to the top of our template:

<head>
    <script type="text/javascript" charset="utf8" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
    <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/1.10.16/css/jquery.dataTables.css">
    <script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.16/js/jquery.dataTables.js"></script>
</head>

Next, at the end of our template we'll add:

<script>$(document).ready(function() {
    $('#fancytable').DataTable();
} );</script>  

Now, we need to give our table the id of "fancytable:"

<table id="fancytable">

We can also give it one of the DataTables classes, for now, I will go with "display"

<table id="fancytable" class="display">

This will give it some over interaction and such. From here, we can use the search box to search for specific terms. Since Donald Trump is the president, there should be many results if you begin to type "Trump." The search is live and immediate, which is neat. Here's the full template code just in case you're missing something:

<head>
    <script type="text/javascript" charset="utf8" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
    <link rel="stylesheet" type="text/css" href="//cdn.datatables.net/1.10.16/css/jquery.dataTables.css">
    <script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.16/js/jquery.dataTables.js"></script>
</head>

<h1>{{.Title}}</h1>

<table id="fancytable" class="display">
    <col width="35%">
    <col width="65%">
    <thead>
        <tr>
            <th>Title</th>
            <th>Keywords</th>
        </tr>
    </thead>
    <tbody>
        {{ range $key, $value := .News }}
         <tr>
            <td><a href="{{ $value.Location }}" target='_blank'>{{ $key }}</td>
            <td>{{ $value.Keyword }}</td>
        </tr>
        {{ end }}
    </tbody>
</table>

<script>$(document).ready(function() {
    $('#fancytable').DataTable();
} );</script>  

Whew, alright, so we've got our first working news aggregator, a proof of concept... but jeez this thing is slow. I personally see ~5000ms. There are two major reasons for the speed here. The first reason for some lag is the passing of all of this data to the browser. We've got almost 1500 rows in this table, lots of keywords...etc. This is an expensive page overall. That said, the *main* draw on our speed is more likely to be the fact that we're visiting each page linearly, in a chain, meaning we request for the first sitemap data, wait for the response, do some stuff, then visit the next sitemap, wait for a response...etc. Those response waits add up. What we'll be talking about in the coming tutorials is options we have for speeding things up.

The next tutorial: Goroutines - Concurrency in Goprogramming