diff --git a/auth.go b/auth.go
index 1de0087..f3ad328 100644
--- a/auth.go
+++ b/auth.go
@@ -9,6 +9,8 @@ import (
 	"net"
 	"strings"
 	"time"
+	"os"
+	"bufio"
 
 	"github.com/shazow/ssh-chat/set"
 	"github.com/shazow/ssh-chat/sshd"
@@ -51,11 +53,14 @@ func newAuthAddr(addr net.Addr) string {
 // If the contained passphrase is not empty, it complements a whitelist.
 type Auth struct {
 	passphraseHash []byte
+	WhitelistMode  bool
 	bannedAddr     *set.Set
 	bannedClient   *set.Set
 	banned         *set.Set
 	whitelist      *set.Set
 	ops            *set.Set
+	opFile string
+	whitelistFile string
 }
 
 // NewAuth creates a new empty Auth.
@@ -82,7 +87,7 @@ func (a *Auth) SetPassphrase(passphrase string) {
 
 // AllowAnonymous determines if anonymous users are permitted.
 func (a *Auth) AllowAnonymous() bool {
-	return a.whitelist.Len() == 0 && a.passphraseHash == nil
+	return !a.WhitelistMode && a.passphraseHash == nil
 }
 
 // AcceptPassphrase determines if passphrase authentication is accepted.
@@ -158,18 +163,38 @@ func (a *Auth) IsOp(key ssh.PublicKey) bool {
 	return a.ops.In(authkey)
 }
 
+// TODO: the *FromFile could be replaced by a single LoadFromFile taking the function (i.e. auth.Op/auth.Whitelist)
+// TODO: consider reloading on empty path
+
+// LoadOpsFromFile reads a file in authorized_keys format and makes public keys operators
+func (a *Auth) LoadOpsFromFile(path string) error {
+	a.opFile = path
+	return fromFile(path, func(key ssh.PublicKey){a.Op(key, 0)})
+}
+
 // Whitelist will set a public key as a whitelisted user.
 func (a *Auth) Whitelist(key ssh.PublicKey, d time.Duration) {
 	if key == nil {
 		return
 	}
+	var err error
 	authItem := newAuthItem(key)
 	if d != 0 {
-		a.whitelist.Set(set.Expire(authItem, d))
+		err = a.whitelist.Set(set.Expire(authItem, d))
 	} else {
-		a.whitelist.Set(authItem)
+		err = a.whitelist.Set(authItem)
 	}
-	logger.Debugf("Added to whitelist: %q (for %s)", authItem.Key(), d)
+	if err == nil {
+		logger.Debugf("Added to whitelist: %q (for %s)", authItem.Key(), d)
+	} else {
+		logger.Errorf("Error adding %q to whitelist for %s: %s", authItem.Key(), d, err)
+	}
+}
+
+// LoadWhitelistFromFile reads a file in authorized_keys format and whitelists public keys
+func (a *Auth) LoadWhitelistFromFile(path string) error {
+	a.whitelistFile = path
+	return fromFile(path, func(key ssh.PublicKey){a.Whitelist(key, 0)})
 }
 
 // Ban will set a public key as banned.
@@ -276,3 +301,29 @@ func (a *Auth) BanQuery(q string) error {
 
 	return nil
 }
+
+func fromFile(path string, handler func(ssh.PublicKey)) error {
+	if path == "" {
+		return nil
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		key, _, _, _, err := ssh.ParseAuthorizedKey(scanner.Bytes())
+		if err != nil {
+			if err.Error() == "ssh: no key found" {
+				// TODO: do we really want to always ignore this?
+				continue // Skip line
+			}
+			return err
+		}
+		handler(key)
+	}
+	return nil
+}
diff --git a/cmd/ssh-chat/cmd.go b/cmd/ssh-chat/cmd.go
index a13e7ff..fd46428 100644
--- a/cmd/ssh-chat/cmd.go
+++ b/cmd/ssh-chat/cmd.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bufio"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -13,7 +12,6 @@ import (
 	"github.com/alexcesaro/log"
 	"github.com/alexcesaro/log/golog"
 	flags "github.com/jessevdk/go-flags"
-	"golang.org/x/crypto/ssh"
 
 	sshchat "github.com/shazow/ssh-chat"
 	"github.com/shazow/ssh-chat/chat"
@@ -138,35 +136,16 @@ func main() {
 		auth.SetPassphrase(options.Passphrase)
 	}
 
-	err = fromFile(options.Admin, func(line []byte) error {
-		key, _, _, _, err := ssh.ParseAuthorizedKey(line)
-		if err != nil {
-			if err.Error() == "ssh: no key found" {
-				return nil // Skip line
-			}
-			return err
-		}
-		auth.Op(key, 0)
-		return nil
-	})
+	err = auth.LoadOpsFromFile(options.Admin)
 	if err != nil {
 		fail(5, "Failed to load admins: %v\n", err)
 	}
 
-	err = fromFile(options.Whitelist, func(line []byte) error {
-		key, _, _, _, err := ssh.ParseAuthorizedKey(line)
-		if err != nil {
-			if err.Error() == "ssh: no key found" {
-				return nil // Skip line
-			}
-			return err
-		}
-		auth.Whitelist(key, 0)
-		return nil
-	})
+	err = auth.LoadWhitelistFromFile(options.Whitelist)
 	if err != nil {
 		fail(6, "Failed to load whitelist: %v\n", err)
 	}
+	auth.WhitelistMode = options.Whitelist != ""
 
 	if options.Motd != "" {
 		host.GetMOTD = func() (string, error) {
@@ -206,25 +185,3 @@ func main() {
 	<-sig // Wait for ^C signal
 	fmt.Fprintln(os.Stderr, "Interrupt signal detected, shutting down.")
 }
-
-func fromFile(path string, handler func(line []byte) error) error {
-	if path == "" {
-		// Skip
-		return nil
-	}
-
-	file, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		err := handler(scanner.Bytes())
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}