Displays the default robots.txt file content.
Source
*
* @since 2.1.0
* @since 5.3.0 Remove the "Disallow: /" output if search engine visibility is
* discouraged in favor of robots meta HTML tag via wp_robots_no_robots()
* filter callback.
*/
function do_robots() {
header( 'Content-Type: text/plain; charset=utf-8' );
/**
* Fires when displaying the robots.txt file.
*
* @since 2.1.0
*/
do_action( 'do_robotstxt' );
$output = "User-agent: *\n";
$public = (bool) get_option( 'blog_public' );
$site_url = parse_url( site_url() );
$path = ( ! empty( $site_url['path'] ) ) ? $site_url['path'] : '';
$output .= "Disallow: $path/wp-admin/\n";
$output .= "Allow: $path/wp-admin/admin-ajax.php\n";
/**
* Filters the robots.txt output.
*
* @since 3.0.0
Changelog
Version | Description |
---|---|
5.3.0 | Remove the "Disallow: /" output if search engine visibility is discouraged in favor of robots meta HTML tag via wp_robots_no_robots() filter callback. |
2.1.0 | Introduced. |
Some examples of how robots.txt file content can be altered.