diff --git a/gplugin/gplugin-loader.c b/gplugin/gplugin-loader.c
--- a/gplugin/gplugin-loader.c
+++ b/gplugin/gplugin-loader.c
@@ -91,6 +91,8 @@
 	GError **error)
 {
 	GPluginLoaderClass *klass = NULL;
+	GPluginPlugin *plugin = NULL;
+	GError *real_error = NULL;
 
 	g_return_val_if_fail(loader != NULL, NULL);
 	g_return_val_if_fail(GPLUGIN_IS_LOADER(loader), NULL);
@@ -99,10 +101,32 @@
 
 	klass = GPLUGIN_LOADER_GET_CLASS(loader);
 
-	if(klass && klass->query)
-		return klass->query(loader, filename, error);
+	if(klass != NULL && klass->query != NULL) {
+		plugin = klass->query(loader, filename, &real_error);
+	}
+
+	if(!GPLUGIN_IS_PLUGIN(plugin)) {
+		if(real_error == NULL) {
+			real_error = g_error_new_literal(
+				GPLUGIN_DOMAIN,
+				0,
+				"Failed to query plugin : unknown");
+		}
 
-	return NULL;
+		g_propagate_error(error, real_error);
+	} else {
+		/* If the plugin successfully queried but returned an error, ignore the
+		 * error.
+		 */
+		g_clear_error(&real_error);
+
+		/* Likewise, make sure the plugin's error is set to NULL. */
+		g_object_set(G_OBJECT(plugin), "error", NULL, NULL);
+
+		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_QUERIED);
+	}
+
+	return plugin;
 }
 
 /**
@@ -123,23 +147,53 @@
 	GError **error)
 {
 	GPluginLoaderClass *klass = NULL;
+	GError *real_error = NULL;
 	gboolean ret = FALSE;
 
 	g_return_val_if_fail(loader != NULL, FALSE);
 	g_return_val_if_fail(GPLUGIN_IS_LOADER(loader), FALSE);
 	g_return_val_if_fail(GPLUGIN_IS_PLUGIN(plugin), FALSE);
 
+	/* if the plugin is already loaded there's nothing for us to do */
+	if(gplugin_plugin_get_state(plugin) == GPLUGIN_PLUGIN_STATE_LOADED) {
+		return TRUE;
+	}
+
 	klass = GPLUGIN_LOADER_GET_CLASS(loader);
 
-	if(klass && klass->load)
-		ret = klass->load(loader, plugin, error);
+	if(klass != NULL && klass->load != NULL) {
+		ret = klass->load(loader, plugin, &real_error);
+	}
+
+	if(!ret) {
+		if(real_error == NULL) {
+			real_error = g_error_new_literal(
+				GPLUGIN_DOMAIN,
+				0,
+				"Failed to load plugin : unknown");
+		}
+
+		/* Set the error on the plugin as well.  This  has to be before we
+		 * propagate the error, because error is invalidate at that point.
+		 */
+		g_object_set(plugin, "error", real_error, NULL);
 
-	if(!ret && error && *error == NULL) {
-		g_set_error(
-			error,
-			GPLUGIN_DOMAIN,
-			0,
-			"Failed to load plugin : unknown");
+		/* Set the state after the error is set, because people might connect
+		 * to the notify signal on the state property.
+		 */
+		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_LOAD_FAILED);
+
+		g_propagate_error(error, real_error);
+	} else {
+		/* If the plugin successfully loaded but returned an error, ignore the
+		 * error.
+		 */
+		g_clear_error(&real_error);
+
+		/* Likewise, make sure the plugin's error is set to NULL. */
+		g_object_set(G_OBJECT(plugin), "error", NULL, NULL);
+
+		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_LOADED);
 	}
 
 	return ret;
@@ -163,23 +217,46 @@
 	GError **error)
 {
 	GPluginLoaderClass *klass = NULL;
+	GError *real_error = NULL;
 	gboolean ret = FALSE;
 
 	g_return_val_if_fail(loader != NULL, FALSE);
 	g_return_val_if_fail(GPLUGIN_IS_LOADER(loader), FALSE);
 	g_return_val_if_fail(GPLUGIN_IS_PLUGIN(plugin), FALSE);
 
+	if(gplugin_plugin_get_state(plugin) != GPLUGIN_PLUGIN_STATE_LOADED) {
+		return TRUE;
+	}
+
 	klass = GPLUGIN_LOADER_GET_CLASS(loader);
 
-	if(klass && klass->unload)
-		ret = klass->unload(loader, plugin, error);
+	if(klass != NULL && klass->unload != NULL) {
+		ret = klass->unload(loader, plugin, &real_error);
+	}
+
+	if(!ret) {
+		if(real_error == NULL) {
+			real_error = g_error_new_literal(
+				GPLUGIN_DOMAIN,
+				0,
+				"Failed to unload plugin : unknown");
+		}
+
+		g_object_set(G_OBJECT(plugin), "error", real_error, NULL);
 
-	if(!ret && error && *error == NULL) {
-		g_set_error(
-			error,
-			GPLUGIN_DOMAIN,
-			0,
-			"Failed to unload plugin : unknown");
+		/* Set the state after the error is set, because people might connect
+		 * to the notify signal on the state property.
+		 */
+		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_UNLOAD_FAILED);
+
+		g_propagate_error(error, real_error);
+	} else {
+		/* If the plugin successfully unloaded but returned an error, ignore the
+		 * error.
+		 */
+		g_clear_error(error);
+
+		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_QUERIED);
 	}
 
 	return ret;
diff --git a/gplugin/gplugin-manager.c b/gplugin/gplugin-manager.c
--- a/gplugin/gplugin-manager.c
+++ b/gplugin/gplugin-manager.c
@@ -701,17 +701,8 @@
 							errors++;
 
 							g_error_free(error);
-						} else {
-							gplugin_plugin_set_state(
-								plugin,
-								GPLUGIN_PLUGIN_STATE_LOADED);
 						}
 					} else {
-						/* finally set the plugin state queried */
-						gplugin_plugin_set_state(
-							plugin,
-							GPLUGIN_PLUGIN_STATE_QUERIED);
-
 						/* if errors is greater than 0 set
 						 * manager->refresh_needed to TRUE.
 						 */
@@ -966,8 +957,9 @@
 	g_return_val_if_fail(GPLUGIN_IS_PLUGIN(plugin), FALSE);
 
 	/* if the plugin is already loaded there's nothing for us to do */
-	if(gplugin_plugin_get_state(plugin) == GPLUGIN_PLUGIN_STATE_LOADED)
+	if(gplugin_plugin_get_state(plugin) == GPLUGIN_PLUGIN_STATE_LOADED) {
 		return TRUE;
+	}
 
 	/* now try to get the plugin info from the plugin */
 	info = gplugin_plugin_get_info(plugin);
@@ -1029,27 +1021,8 @@
 
 	ret = gplugin_loader_load_plugin(loader, plugin, &error);
 	if(ret) {
-		/* If the plugin successfully loaded but returned an error, ignore the
-		 * error.
-		 */
-		g_clear_error(&error);
-
-		/* Likewise, make sure the plugin's error is set to NULL. */
-		g_object_set(G_OBJECT(plugin), "error", NULL, NULL);
-
-		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_LOADED);
 		g_signal_emit(manager, signals[SIG_LOADED], 0, plugin);
 	} else {
-		/* Set the error on the plugin as well.  This  has to be before we
-		 * propagate the error, because error is invalidate at that point.
-		 */
-		g_object_set(G_OBJECT(plugin), "error", error, NULL);
-
-		/* Set the state after the error is set, because people might connect
-		 * to the notify signal on the state property.
-		 */
-		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_LOAD_FAILED);
-
 		g_signal_emit(manager, signals[SIG_LOAD_FAILED], 0, plugin);
 
 		g_propagate_error(ret_error, error);
@@ -1070,8 +1043,9 @@
 
 	g_return_val_if_fail(GPLUGIN_IS_PLUGIN(plugin), FALSE);
 
-	if(gplugin_plugin_get_state(plugin) != GPLUGIN_PLUGIN_STATE_LOADED)
+	if(gplugin_plugin_get_state(plugin) != GPLUGIN_PLUGIN_STATE_LOADED) {
 		return TRUE;
+	}
 
 	loader = gplugin_plugin_get_loader(plugin);
 	if(!GPLUGIN_IS_LOADER(loader)) {
@@ -1091,32 +1065,15 @@
 
 		g_propagate_error(ret_error, error);
 
+		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_LOAD_FAILED);
+
 		return ret;
 	}
 
 	ret = gplugin_loader_unload_plugin(loader, plugin, &error);
 	if(ret) {
-		/* If the plugin successfully loaded but returned an error, ignore the
-		 * error.
-		 */
-		g_clear_error(&error);
-
-		/* Like, make sure the plugin's error is set to NULL. */
-		g_object_set(G_OBJECT(plugin), "error", NULL, NULL);
-
-		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_QUERIED);
 		g_signal_emit(manager, signals[SIG_UNLOADED], 0, plugin);
 	} else {
-		/* Set the error on the plugin as well.  This  has to be before we
-		 * propagate the error, because error is invalidate at that point.
-		 */
-		g_object_set(G_OBJECT(plugin), "error", error, NULL);
-
-		/* Set the state after the error is set, because people might connect
-		 * to the notify signal on the state property.
-		 */
-		gplugin_plugin_set_state(plugin, GPLUGIN_PLUGIN_STATE_UNLOAD_FAILED);
-
 		g_signal_emit(manager, signals[SIG_UNLOAD_FAILED], 0, plugin);
 
 		g_propagate_error(ret_error, error);
