diff --git a/src/world/map.c b/src/world/map.c
index 85b9c42d59..a040e39f18 100644
--- a/src/world/map.c
+++ b/src/world/map.c
@@ -765,6 +765,10 @@ void game_command_remove_large_scenery(int* eax, int* ebx, int* ecx, int* edx, i
 		if ((map_element->type & MAP_ELEMENT_DIRECTION_MASK) != map_element_direction)
 			continue;
 
+		// If we are removing ghost elements
+		if((*ebx & 0x40) && !(map_element->flags & MAP_ELEMENT_FLAG_GHOST))
+			continue;
+		
 		element_found = 1;
 		break;
 	} while (!map_element_is_last_for_tile(map_element++));
@@ -774,11 +778,6 @@ void game_command_remove_large_scenery(int* eax, int* ebx, int* ecx, int* edx, i
 		return;
 	}
 
-	if((*ebx & 0x40) && !(map_element->flags & MAP_ELEMENT_FLAG_GHOST)){
-		*ebx = 0;
-		return;
-	}
-
 	map_element_remove_banner_entry(map_element);
 	
 	int ecx2 = map_element->properties.scenerymultiple.type >> 10;
@@ -835,6 +834,10 @@ void game_command_remove_large_scenery(int* eax, int* ebx, int* ecx, int* edx, i
 			if (sceneryElement->base_height != currentTile.z / 8)
 				continue;
 
+			// If we are removing ghost elements
+			if ((*ebx & 0x40) && !(sceneryElement->flags & MAP_ELEMENT_FLAG_GHOST))
+				continue;
+
 			map_invalidate_tile_full(currentTile.x, currentTile.y);
 			map_element_remove(sceneryElement);
 			tile_not_found = 0;