Example migrated from Codex:

Creating a shortcode and a tag button

First, create a plugin that registers the buttons and their JavaScript.

/* Plugin Name: My TinyMCE Buttons */
add_action( 'admin_init', 'my_tinymce_button' );

function my_tinymce_button() {
     if ( current_user_can( 'edit_posts' ) && current_user_can( 'edit_pages' ) ) {
          add_filter( 'mce_buttons', 'my_register_tinymce_button' );
          add_filter( 'mce_external_plugins', 'my_add_tinymce_button' );
     }
}

function my_register_tinymce_button( $buttons ) {
     array_push( $buttons, "button_eek", "button_green" );
     return $buttons;
}

function my_add_tinymce_button( $plugin_array ) {
     $plugin_array['my_button_script'] = plugins_url( '/mybuttons.js', __FILE__ ) ;
     return $plugin_array;
}

Then, in the same folder as the plugin, the JavaScript file mybuttons.js. We add our two buttons (eek and green), one is a direct onclick that inserts a Shortcode and the other is a command that adds an h3 tag to the selected text in the editor.

(function() {
     /* Register the buttons */
     tinymce.create('tinymce.plugins.MyButtons', {
          init : function(ed, url) {
               /**
               * Inserts shortcode content
               */
               ed.addButton( 'button_eek', {
                    title : 'Insert shortcode',
                    image : '../wp-includes/images/smilies/icon_eek.gif',
                    onclick : function() {
                         ed.selection.setContent('[myshortcode]');
                    }
               });
               /**
               * Adds HTML tag to selected content
               */
               ed.addButton( 'button_green', {
                    title : 'Add span',
                    image : '../wp-includes/images/smilies/icon_mrgreen.gif',
                    cmd: 'button_green_cmd'
               });
               ed.addCommand( 'button_green_cmd', function() {
                    var selected_text = ed.selection.getContent();
                    var return_text = '';
                    return_text = '<h1>' + selected_text + '</h1>';
                    ed.execCommand('mceInsertContent', 0, return_text);
               });
          },
          createControl : function(n, cm) {
               return null;
          },
     });
     /* Start the buttons */
     tinymce.PluginManager.add( 'my_button_script', tinymce.plugins.MyButtons );
})();

You can pass PHP values to the JavaScript file printing directly in admin_head:

foreach ( array('post.php','post-new.php') as $hook ) {
     add_action( "admin_head-$hook", 'my_admin_head' );
}

/**
 * Localize Script
 */
function my_admin_head() {
    $plugin_url = plugins_url( '/', __FILE__ );
    ?>
<!-- TinyMCE Shortcode Plugin -->
<script type='text/javascript'>
var my_plugin = {
    'url': '<?php echo $plugin_url; ?>',
};
</script>
<!-- TinyMCE Shortcode Plugin -->
    <?php
}

And then, access it in mybuttons.js with console.log(my_plugin.url);.