diff --git a/cmd/ssh-chat/cmd.go b/cmd/ssh-chat/cmd.go
index ac7639a..23db63e 100644
--- a/cmd/ssh-chat/cmd.go
+++ b/cmd/ssh-chat/cmd.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -9,6 +10,7 @@ import (
 	"os/signal"
 	"os/user"
 	"strings"
+	"time"
 
 	"github.com/alexcesaro/log"
 	"github.com/alexcesaro/log/golog"
@@ -19,23 +21,25 @@ import (
 	"github.com/shazow/ssh-chat/chat"
 	"github.com/shazow/ssh-chat/chat/message"
 	"github.com/shazow/ssh-chat/sshd"
+
+	_ "net/http/pprof"
 )
-import _ "net/http/pprof"
 
 // Version of the binary, assigned during build.
 var Version string = "dev"
 
 // Options contains the flag options
 type Options struct {
-	Verbose   []bool `short:"v" long:"verbose" description:"Show verbose logging."`
-	Version   bool   `long:"version" description:"Print version and exit."`
-	Identity  string `short:"i" long:"identity" description:"Private key to identify server with." default:"~/.ssh/id_rsa"`
-	Bind      string `long:"bind" description:"Host and port to listen on." default:"0.0.0.0:2022"`
-	Admin     string `long:"admin" description:"File of public keys who are admins."`
-	Whitelist string `long:"whitelist" description:"Optional file of public keys who are allowed to connect."`
-	Motd      string `long:"motd" description:"Optional Message of the Day file."`
-	Log       string `long:"log" description:"Write chat log to this file."`
-	Pprof     int    `long:"pprof" description:"Enable pprof http server for profiling."`
+	Verbose    []bool `short:"v" long:"verbose" description:"Show verbose logging."`
+	Version    bool   `long:"version" description:"Print version and exit."`
+	Identity   string `short:"i" long:"identity" description:"Private key to identify server with." default:"~/.ssh/id_rsa"`
+	Bind       string `long:"bind" description:"Host and port to listen on." default:"0.0.0.0:2022"`
+	Admin      string `long:"admin" description:"File of public keys who are admins."`
+	Whitelist  string `long:"whitelist" description:"Optional file of public keys who are allowed to connect."`
+	Passphrase string `long:"unsafe-passphrase" description:"Require an interactive passphrase to connect. Whitelist feature is more secure."`
+	Motd       string `long:"motd" description:"Optional Message of the Day file."`
+	Log        string `long:"log" description:"Write chat log to this file."`
+	Pprof      int    `long:"pprof" description:"Enable pprof http server for profiling."`
 }
 
 var logLevels = []log.Level{
@@ -110,6 +114,28 @@ func main() {
 	config.AddHostKey(signer)
 	config.ServerVersion = "SSH-2.0-Go ssh-chat"
 
+	if options.Passphrase != "" {
+		cb := config.KeyboardInteractiveCallback
+		config.KeyboardInteractiveCallback = func(conn ssh.ConnMetadata, challenge ssh.KeyboardInteractiveChallenge) (*ssh.Permissions, error) {
+			perm, err := cb(conn, challenge)
+			if err != nil {
+				return perm, err
+			}
+			answers, err := challenge("", "", []string{"Passphrase required to connect: "}, []bool{true})
+			if err != nil {
+				return nil, err
+			}
+			if len(answers) == 1 && answers[0] == options.Passphrase {
+				// Success
+				return perm, nil
+			}
+			// It's not gonna do much but may as well throttle brute force attempts a little
+			time.Sleep(2 * time.Second)
+
+			return nil, errors.New("incorrect passphrase")
+		}
+	}
+
 	s, err := sshd.ListenSSH(options.Bind, config)
 	if err != nil {
 		fail(4, "Failed to listen on socket: %v\n", err)
diff --git a/sshd/auth.go b/sshd/auth.go
index afa7271..a140413 100644
--- a/sshd/auth.go
+++ b/sshd/auth.go
@@ -19,6 +19,7 @@ type Auth interface {
 }
 
 // MakeAuth makes an ssh.ServerConfig which performs authentication against an Auth implementation.
+// TODO: Switch to using ssh.AuthMethod instead?
 func MakeAuth(auth Auth) *ssh.ServerConfig {
 	config := ssh.ServerConfig{
 		NoClientAuth: false,