diff --git a/pbx/pbx_config.c b/pbx/pbx_config.c
index 4c7c42ae2ac96e53cc614006a834ca2ede989fcd..8596264ca2b4e70becbb20a7cd42534f09e2c174 100644
--- a/pbx/pbx_config.c
+++ b/pbx/pbx_config.c
@@ -1543,15 +1543,12 @@ static void pbx_load_users(void)
 	int len;
 	int hasvoicemail;
 	int start, finish, x;
-	struct ast_context *con;
+	struct ast_context *con = NULL;
 	struct ast_flags config_flags = { 0 };
 	
 	cfg = ast_config_load("users.conf", config_flags);
 	if (!cfg)
 		return;
-	con = ast_context_find_or_create(&local_contexts, userscontext, registrar);
-	if (!con)
-		return;
 
 	for (cat = ast_category_browse(cfg, NULL); cat ; cat = ast_category_browse(cfg, cat)) {
 		if (!strcasecmp(cat, "general"))
@@ -1603,6 +1600,16 @@ static void pbx_load_users(void)
 			}
 		}
 		if (!ast_strlen_zero(iface)) {
+			/* Only create a context here when it is really needed. Otherwise default empty context
+			created by pbx_config may conflict with the one explicitly created by pbx_ael */
+			if (!con)
+				con = ast_context_find_or_create(&local_contexts, userscontext, registrar);
+
+			if (!con) {
+				ast_log(LOG_ERROR, "Can't find/create user context '%s'\n", userscontext);
+				return;
+			}
+
 			/* Add hint */
 			ast_add_extension2(con, 0, cat, -1, NULL, NULL, iface, NULL, NULL, registrar);
 			/* If voicemail, use "stdexten" else use plain old dial */