Edit: I've tried added it to _getProductCollection in List.php as suggested by Mohit. This removes the product as expected but unfortunately still leaves a gap. I'll leave the code for that here as well.
protected function _getProductCollection()
{
if (is_null($this->_productCollection)) {
$layer = $this->getLayer();
/* @var $layer Mage_Catalog_Model_Layer */
if ($this->getShowRootCategory()) {
$this->setCategoryId(Mage::app()->getStore()->getRootCategoryId());
}
// if this is a product view page
if (Mage::registry('product')) {
// get collection of categories this product is associated with
$categories = Mage::registry('product')->getCategoryCollection()
->setPage(1, 1)
->load();
// if the product is associated with any category
if ($categories->count()) {
// show products from this category
$this->setCategoryId(current($categories->getIterator()));
}
}
$origCategory = null;
if ($this->getCategoryId()) {
$category = Mage::getModel('catalog/category')->load($this->getCategoryId());
if ($category->getId()) {
$origCategory = $layer->getCurrentCategory();
$layer->setCurrentCategory($category);
}
}
$this->_productCollection = $layer->getProductCollection();
$this->prepareSortableFieldsByCategory($layer->getCurrentCategory());
if ($origCategory) {
$layer->setCurrentCategory($origCategory);
}
}
//Try and remove a specific ID from the collection
$collection = $this->_productCollection;
$collection->removeItemByKey(4146);
return $collection;
}