diff --git a/server/user.c b/server/user.c
index c4a16cc6d3d610ffb8c91153b3e8c24f415694d4..b4f3e0e571c8da2fbe4455174a30693cee71f161 100644
--- a/server/user.c
+++ b/server/user.c
@@ -146,8 +146,9 @@ void *next_user_handle( user_handle_t *handle, enum user_object type )
     if (!*handle) entry = handles;
     else
     {
-        if (!(entry = handle_to_entry( *handle ))) return NULL;
-        entry++;  /* start from the next one */
+        int index = (*handle & 0xffff) - FIRST_USER_HANDLE;
+        if (index < 0 || index >= nb_handles) return NULL;
+        entry = handles + index + 1;  /* start from the next one */
     }
     while (entry < handles + nb_handles)
     {