apply_filters( ‘robots_txt’, string $output, bool $public )

User Contributed Notes

1. Skip to note 2 content

   wp_kc 5 years ago

   You must log in to vote on the helpfulness of this noteVote results for this note: 2You must log in to vote on the helpfulness of this note

   An example of how robots.txt file content can be altered.

   /**
    * Add Disallow for some file types.
    * Add "Disallow: /wp-login.php/\n".
    * Remove "Allow: /wp-admin/admin-ajax.php\n".
    * Calculate and add a "Sitemap:" link.
    */
   add_filter( 'robots_txt', function( $output, $public ) {
   	/**
   	 * If "Search engine visibility" is disabled,
   	 * strongly tell all robots to go away.
   	 */
   	if ( '0' == $public ) {
   
   		$output = "User-agent: *\nDisallow: /\nDisallow: /*\nDisallow: /*?\n";
   
   	} else {
   
   		/**
   		 * Get site path.
   		 */
   		$site_url = parse_url( site_url() );
   		$path	  = ( ! empty( $site_url[ 'path' ] ) ) ? $site_url[ 'path' ] : '';
   
   		/**
   		 * Add new disallow.
   		 */
   		$output .= "Disallow: $path/wp-login.php\n";
   
   		/**
   		 * Disallow some file types
   		 */
   		foreach(['jpeg','jpg','gif','png','mp4','webm','woff','woff2','ttf','eot'] as $ext){
   			$output .= "Disallow: /*.{$ext}$\n";
   		}
   
   		/**
   		 * Remove line that allows robots to access AJAX interface.
   		 */
   		$robots = preg_replace( '/Allow: [^\0\s]*\/wp-admin\/admin-ajax\.php\n/', '', $output );
   
   		/**
   		 * If no error occurred, replace $output with modified value.
   		 */
   		if ( null !== $robots ) {
   			$output = $robots;
   		}
   		/**
   		 * Calculate and add a "Sitemap:" link.
   		 * Modify as needed.
   		 */
   		$output .= "Sitemap: {$site_url[ 'scheme' ]}://{$site_url[ 'host' ]}/sitemap_index.xml\n";
   	}
   
   	return $output;
   
   }, 99, 2 );  // Priority 99, Number of Arguments 2.