How to test http calls in go using httptest

Question

I have the following code:

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "net/http"
    "time"
)

type twitterResult struct {
    Results []struct {
        Text     string `json:"text"`
        Ids      string `json:"id_str"`
        Name     string `json:"from_user_name"`
        Username string `json:"from_user"`
        UserId   string `json:"from_user_id_str"`
    }
}

var (
  twitterUrl = "http://search.twitter.com/search.json?q=%23UCL"
  pauseDuration = 5 * time.Second
)

func retrieveTweets(c chan<- *twitterResult) {
    for {
        resp, err := http.Get(twitterUrl)
        if err != nil {
            log.Fatal(err)
        }

        defer resp.Body.Close()
        body, err := ioutil.ReadAll(resp.Body)
        r := new(twitterResult) //or &twitterResult{} which returns *twitterResult
        err = json.Unmarshal(body, &r)
        if err != nil {
            log.Fatal(err)
        }
        c <- r
        time.Sleep(pauseDuration)
    }

}

func displayTweets(c chan *twitterResult) {
    tweets := <-c
    for _, v := range tweets.Results {
        fmt.Printf("%v:%v\n", v.Username, v.Text)
    }

}

func main() {
    c := make(chan *twitterResult)
    go retrieveTweets(c)
    for {
        displayTweets(c)
    }

}

I'd like to write some tests for it, but I'm not sure how to use the httptest package http://golang.org/pkg/net/http/httptest/ would appreciate some pointers

I came up with this (shamelessly copied from the tests for go OAuth https://code.google.com/p/goauth2/source/browse/oauth/oauth_test.go):

var request = struct {
    path, query       string // request
    contenttype, body string // response
}{
    path:        "/search.json?",
    query:       "q=%23Kenya",
    contenttype: "application/json",
    body:        twitterResponse,
}

var (
    twitterResponse = `{ 'results': [{'text':'hello','id_str':'34455w4','from_user_name':'bob','from_user_id_str':'345424'}]}`
)

func TestRetrieveTweets(t *testing.T) {
    handler := func(w http.ResponseWriter, r *http.Request) {

        w.Header().Set("Content-Type", request.contenttype)
        io.WriteString(w, request.body)
    }

    server := httptest.NewServer(http.HandlerFunc(handler))
    defer server.Close()

    resp, err := http.Get(server.URL)
    if err != nil {
        t.Fatalf("Get: %v", err)
    }
    checkBody(t, resp, twitterResponse)
}

func checkBody(t *testing.T, r *http.Response, body string) {
    b, err := ioutil.ReadAll(r.Body)
    if err != nil {
        t.Error("reading reponse body: %v, want %q", err, body)
    }
    if g, w := string(b), body; g != w {
        t.Errorf("request body mismatch: got %q, want %q", g, w)
    }
}

Answer 1

myserver_test.go

package myserver

import (
  "fmt"
  "io/ioutil"
  "net/http"
  "net/http/httptest"
  "testing"
)

func TestMyHandler(t *testing.T) {
  handler := &MyHandler{}
  server := httptest.NewServer(handler)
  defer server.Close()

  for _, i := range []int{1, 2} {
      resp, err := http.Get(server.URL)
      if err != nil {
          t.Fatal(err)
      }
      if resp.StatusCode != 200 {
          t.Fatalf("Received non-200 response: %d\n", resp.StatusCode)
      }
      expected := fmt.Sprintf("Visitor count: %d.", i)
      actual, err := ioutil.ReadAll(resp.Body)
      if err != nil {
          t.Fatal(err)
      }
      if expected != string(actual) {
          t.Errorf("Expected the message '%s'\n", expected)
      }
  }
}

myserver.go

package myserver

import (
  "fmt"
  "net/http"
  "sync"
)

type MyHandler struct {
  sync.Mutex
  count int
}

func (h *MyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
  var count int
  h.Lock()
  h.count++
  count = h.count
  h.Unlock()

  fmt.Fprintf(w, "Visitor count: %d.", count)
}

Answer 2

httptest does two types of tests: response and server

Response test:

func TestHeader3D(t *testing.T) {
    resp := httptest.NewRecorder()

    uri := "/3D/header/?"
    path := "/home/test"
    unlno := "997225821"

    param := make(url.Values)
    param["param1"] = []string{path}
    param["param2"] = []string{unlno}

    req, err := http.NewRequest("GET", uri+param.Encode(), nil)
    if err != nil {
            t.Fatal(err)
    }

    http.DefaultServeMux.ServeHTTP(resp, req)
    if p, err := ioutil.ReadAll(resp.Body); err != nil {
            t.Fail()
    } else {
            if strings.Contains(string(p), "Error") {
                    t.Errorf("header response shouldn't return error: %s", p)
            } else if !strings.Contains(string(p), `expected result`) {
                    t.Errorf("header response doen't match:\n%s", p)
            }
    }
}

Server test (which is what you need to use):

func TestIt(t *testing.T){
    ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        w.Header().Set("Content-Type", "application/json")
        fmt.Fprintln(w, `{"fake twitter json string"}`)
    }))
    defer ts.Close()

    twitterUrl = ts.URL
    c := make(chan *twitterResult)
    go retrieveTweets(c)

    tweet := <-c
    if tweet != expected1 {
        t.Fail()
    }
    tweet = <-c
    if tweet != expected2 {
        t.Fail()
    }
}

BTW, you don't need to pass in the pointer of r, because it's already a pointer.

err = json.Unmarshal(body, r)

EDIT: for my recorder test, I could use my http handler like this:

handler(resp, req)

But my original code is not using the default mux (but from Gorilla/mux), and I have some wrapping around the mux, e.g. insert server logging, and adding request context (Gorilla/context), so I had to start from mux and call ServeHTTP

Answer 3

If you want to test your program, it is often best to write it with testing in mind. For instance, if you extracted the inner loop of your retrieveTweets function into something like this:

func downloadTweets(tweetsUrl string) (*twitterResult, error)

You could invoke it with the URL of a test server you've set up using the httptest package without having to worry about the sleeps or repeated requests.