diff --git a/vnc.c b/vnc.c
index a50ee0c..a57516c 100644
--- a/vnc.c
+++ b/vnc.c
@@ -215,9 +215,11 @@ static inline uint32_t vnc_has_feature(VncState *vs, int feature) {
    3) resolutions > 1024
 */
 
-static void vnc_update_client(void *opaque);
+static void vnc_update_client(VncState *vs);
 static void vnc_disconnect_start(VncState *vs);
 static void vnc_disconnect_finish(VncState *vs);
+static void vnc_init_timer(VncDisplay *vd);
+static void vnc_remove_timer(VncDisplay *vd);
 
 static void vnc_colordepth(VncState *vs);
 
@@ -723,9 +725,8 @@ static int find_and_clear_dirty_height(struct VncSurface *s,
     return h;
 }
 
-static void vnc_update_client(void *opaque)
+static void vnc_update_client(VncState *vs)
 {
-    VncState *vs = opaque;
     if (vs->need_update && vs->csock != -1) {
         int y;
         uint8_t *guest_row;
@@ -736,13 +737,9 @@ static void vnc_update_client(void *opaque)
         int saved_offset;
         int has_dirty = 0;
 
-        if (vs->output.offset && !vs->audio_cap && !vs->force_update) {
+        if (vs->output.offset && !vs->audio_cap && !vs->force_update)
             /* kernel send buffers are full -> drop frames to throttle */
-            qemu_mod_timer(vs->timer, qemu_get_clock(rt_clock) + VNC_REFRESH_INTERVAL);
             return;
-        }
-
-        vga_hw_update();
 
         /*
          * Walk through the guest dirty map.
@@ -778,10 +775,8 @@ static void vnc_update_client(void *opaque)
             server_row += ds_get_linesize(vs->ds);
         }
 
-        if (!has_dirty && !vs->audio_cap && !vs->force_update) {
-            qemu_mod_timer(vs->timer, qemu_get_clock(rt_clock) + VNC_REFRESH_INTERVAL);
+        if (!has_dirty && !vs->audio_cap && !vs->force_update)
             return;
-        }
 
         /*
          * Send screen updates to the vnc client using the server
@@ -826,12 +821,8 @@ static void vnc_update_client(void *opaque)
 
     }
 
-    if (vs->csock != -1) {
-        qemu_mod_timer(vs->timer, qemu_get_clock(rt_clock) + VNC_REFRESH_INTERVAL);
-    } else {
+    if (vs->csock == -1)
         vnc_disconnect_finish(vs);
-    }
-
 }
 
 /* audio */
@@ -911,8 +902,6 @@ static void vnc_disconnect_start(VncState *vs)
 
 static void vnc_disconnect_finish(VncState *vs)
 {
-    qemu_del_timer(vs->timer);
-    qemu_free_timer(vs->timer);
     if (vs->input.buffer) qemu_free(vs->input.buffer);
     if (vs->output.buffer) qemu_free(vs->output.buffer);
 #ifdef CONFIG_VNC_TLS
@@ -941,6 +930,7 @@ static void vnc_disconnect_finish(VncState *vs)
     qemu_free(vs->server.ds);
     qemu_free(vs->guest.ds);
     qemu_free(vs);
+    vnc_remove_timer(vs->vd);
 }
 
 int vnc_client_io_error(VncState *vs, int ret, int last_errno)
@@ -2077,6 +2067,38 @@ static int protocol_version(VncState *vs, uint8_t *version, size_t len)
     return 0;
 }
 
+static void vnc_refresh(void *opaque)
+{
+    VncDisplay *vd = opaque;
+    VncState *vs = vd->clients;
+
+    vga_hw_update();
+
+    while (vs != NULL) {
+        vnc_update_client(vs);
+        vs = vs->next;
+    }
+
+    qemu_mod_timer(vd->timer, qemu_get_clock(rt_clock) + VNC_REFRESH_INTERVAL);
+}
+
+static void vnc_init_timer(VncDisplay *vd)
+{
+    if (vd->timer == NULL && vd->clients != NULL) {
+        vd->timer = qemu_new_timer(rt_clock, vnc_refresh, vd);
+        qemu_mod_timer(vd->timer, qemu_get_clock(rt_clock) + VNC_REFRESH_INTERVAL);
+    }
+}
+
+static void vnc_remove_timer(VncDisplay *vd)
+{
+    if (vd->timer != NULL && vd->clients == NULL) {
+        qemu_del_timer(vd->timer);
+        qemu_free_timer(vd->timer);
+        vd->timer = NULL;
+    }
+}
+
 static void vnc_connect(VncDisplay *vd, int csock)
 {
     VncState *vs = qemu_mallocz(sizeof(VncState));
@@ -2089,7 +2111,6 @@ static void vnc_connect(VncDisplay *vd, int csock)
 
     vs->vd = vd;
     vs->ds = vd->ds;
-    vs->timer = qemu_new_timer(rt_clock, vnc_update_client, vs);
     vs->last_x = -1;
     vs->last_y = -1;
 
@@ -2107,6 +2128,7 @@ static void vnc_connect(VncDisplay *vd, int csock)
     vs->next = vd->clients;
     vd->clients = vs;
 
+    vnc_init_timer(vd);
     vnc_update_client(vs);
     /* vs might be free()ed here */
 }
diff --git a/vnc.h b/vnc.h
index 3ae95f3..a35b378 100644
--- a/vnc.h
+++ b/vnc.h
@@ -87,6 +87,7 @@ typedef struct VncDisplay VncDisplay;
 
 struct VncDisplay
 {
+    QEMUTimer *timer;
     int lsock;
     DisplayState *ds;
     VncState *clients;
@@ -112,7 +113,6 @@ struct VncSurface
 
 struct VncState
 {
-    QEMUTimer *timer;
     int csock;
 
     DisplayState *ds;
